From c35669386d9b04bf116c8434b19847f21b11941f Mon Sep 17 00:00:00 2001 From: Auto Git Date: Tue, 4 Oct 2016 11:18:44 -0600 Subject: [PATCH 001/312] Update version to 6.1 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 9abb40e38b..bcc1f20bec 100644 --- a/configure.ac +++ b/configure.ac @@ -33,7 +33,7 @@ dnl added library functionality to testing dnl by John Rosenquist (jrosenquist@adaptivecomputing.com) AC_PREREQ(2.53) -AC_INIT([torque], [master], [torqueusers@supercluster.org]) +AC_INIT([torque], [6.1], [torqueusers@supercluster.org]) AC_REVISION($Revision$) AC_CONFIG_SRCDIR([src/cmds/qrun.c]) AC_CONFIG_AUX_DIR([buildutils]) From a6a0a2ae38e0d45ff571f747c6c050d0b9499add Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 4 Oct 2016 11:35:06 -0600 Subject: [PATCH 002/312] TRQ-3788. Fix an issue with resource min/max enforcement. --- src/lib/Libattr/attr_fn_resc.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/lib/Libattr/attr_fn_resc.c b/src/lib/Libattr/attr_fn_resc.c index bfab5da094..7f4b6595e1 100644 --- a/src/lib/Libattr/attr_fn_resc.c +++ b/src/lib/Libattr/attr_fn_resc.c @@ -402,6 +402,7 @@ int __comp_work__( int &greater_than, int &less_than, bool check_default, + bool compare_forwards, char *EMsg) { @@ -421,7 +422,13 @@ int __comp_work__( { if (atresc->rs_value.at_flags & ATR_VFLAG_SET) { - int rc = atresc->rs_defin->rs_comp(&atresc->rs_value, &r.rs_value); + int rc; + + if (compare_forwards) + rc = atresc->rs_defin->rs_comp(&atresc->rs_value, &r.rs_value); + else + rc = atresc->rs_defin->rs_comp(&r.rs_value, &atresc->rs_value); + if (rc > 0) { if ((EMsg != NULL) && @@ -440,7 +447,7 @@ int __comp_work__( } /* END while() */ return(0); - } // END comp_work() + } // END __comp_work__() @@ -495,7 +502,7 @@ int comp_resc( pbs_attribute. If the queue has a min requirement of resource X and the job has no value set for this resource, this routine will not trigger comp_resc_lt */ - __comp_work__(attr, with, comp_resc_gt, comp_resc_lt, false, NULL); + __comp_work__(attr, with, comp_resc_gt, comp_resc_lt, false, true, NULL); return(0); } /* END comp_resc() */ @@ -561,7 +568,7 @@ int comp_resc2( if (IsQueueCentric == 1) { /* comparison is queue centric */ - __comp_work__(queue_attr, job_attr, comp_resc_gt, comp_resc_lt, true, EMsg); + __comp_work__(queue_attr, job_attr, local_gt, local_lt, true, true, EMsg); } else { @@ -571,7 +578,7 @@ int comp_resc2( pbs_attribute. If the queue has a min requirement of resource X and the job has no value set for this resource, this routine will not trigger comp_resc_lt */ - __comp_work__(job_attr, queue_attr, comp_resc_gt, comp_resc_lt, false, EMsg); + __comp_work__(job_attr, queue_attr, local_gt, local_lt, false, false, EMsg); } if (type == GREATER) From ff0c122813f0c7ba7fc3633bf6ad677eeda91d87 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 4 Oct 2016 15:48:41 -0600 Subject: [PATCH 003/312] TRQ-3788. Fix a display error for qstat -r. --- src/cmds/qstat.c | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/cmds/qstat.c b/src/cmds/qstat.c index 8dd9ef1a26..e6d1e936a5 100644 --- a/src/cmds/qstat.c +++ b/src/cmds/qstat.c @@ -849,6 +849,11 @@ static void altdsp_statjob( rem_walltime = 0; bool dummyProcVal = false; + // Make the memory take the alphabetically last specification. There is no reason it has + // to be this way or should be this way, but I'm making this change so that the behavior + // stays the same. + char mem_char = 'a'; + pat = pstat->attribs; while (pat != NULL) @@ -924,23 +929,48 @@ static void altdsp_statjob( } else if (!strcmp(pat->resource, "mem")) { - snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + if (mem_char < 'm') + { + mem_char = 'm'; + + snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + } } else if (!strcmp(pat->resource, "pmem")) { - snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + if (mem_char < 'p') + { + mem_char = 'p'; + + snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + } } else if (!strcmp(pat->resource, "dmem")) { - snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + if (mem_char < 'd') + { + mem_char = 'd'; + + snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + } } else if (!strcmp(pat->resource, "vmem")) { - snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + if (mem_char < 'v') + { + mem_char = 'v'; + + snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + } } else if (!strcmp(pat->resource, "pvmem")) { - snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + if (mem_char <= 'p') + { + mem_char = 'p'; + + snprintf(rqmem, sizeof(rqmem), "%s", cnv_size(pat->value, alt_opt)); + } } else if (!strcmp(pat->resource, "walltime")) { From 6b2ad7fa2476822f7b42ac792fa8442a32ad3018 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 5 Oct 2016 10:51:57 -0600 Subject: [PATCH 004/312] Make Torque handle versions like 6.1 as well as 6.1.0 --- src/server/pbsnode.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/pbsnode.cpp b/src/server/pbsnode.cpp index adef3a554c..153dd0838a 100644 --- a/src/server/pbsnode.cpp +++ b/src/server/pbsnode.cpp @@ -813,10 +813,10 @@ void pbsnode::set_version( { ptr++; version += strtol(ptr, &ptr, 10); - - // Only set the version if we had the correct format - this->nd_version = version; } + + // Pre-release, we'll have versions like 6.1 instead of 6.1.x + this->nd_version = version; } else if (!strcmp(version_str, "master")) { From 513ad8f99388d76810b5d765b2bb6b83e48dc212 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 6 Oct 2016 10:34:07 -0600 Subject: [PATCH 005/312] TRQ-3750. Fix a memory leak around asynchronous job starts. --- src/lib/Libifl/dec_svrattrl.c | 2 +- src/server/process_request.c | 23 +++++------------------ src/server/req_runjob.c | 13 +------------ 3 files changed, 7 insertions(+), 31 deletions(-) diff --git a/src/lib/Libifl/dec_svrattrl.c b/src/lib/Libifl/dec_svrattrl.c index fd23312cae..f44e2ba3fd 100644 --- a/src/lib/Libifl/dec_svrattrl.c +++ b/src/lib/Libifl/dec_svrattrl.c @@ -178,7 +178,7 @@ int decode_DIS_svrattrl( hasresc = disrui(chan, &rc); if (rc) - break; + break; if (hasresc) { diff --git a/src/server/process_request.c b/src/server/process_request.c index 356fd84ea0..4d4f43b578 100644 --- a/src/server/process_request.c +++ b/src/server/process_request.c @@ -717,8 +717,8 @@ int process_request( int dispatch_request( - int sfds, /* I */ - struct batch_request *request) /* I */ + int sfds, /* I */ + batch_request *request) /* I */ { int rc = PBSE_NONE; @@ -898,8 +898,8 @@ int dispatch_request( break; case PBS_BATCH_RunJob: - case PBS_BATCH_AsyrunJob: + globalset_del_sock(request->rq_conn); rc = req_runjob(request); @@ -1042,7 +1042,7 @@ struct batch_request *alloc_br( struct batch_request *req = NULL; - if ((req = (struct batch_request *)calloc(1, sizeof(struct batch_request))) == NULL) + if ((req = (batch_request *)calloc(1, sizeof(batch_request))) == NULL) { log_err(errno, __func__, msg_err_malloc); } @@ -1186,18 +1186,8 @@ void free_br( break; case PBS_BATCH_MvJobFile: - - case PBS_BATCH_jobscript2: - - if (preq->rq_ind.rq_jobfile.rq_data) - { - free(preq->rq_ind.rq_jobfile.rq_data); - preq->rq_ind.rq_jobfile.rq_data = NULL; - } - break; - - case PBS_BATCH_jobscript: + case PBS_BATCH_jobscript2: if (preq->rq_ind.rq_jobfile.rq_data) { @@ -1299,7 +1289,6 @@ void free_br( break; case PBS_BATCH_RunJob: - case PBS_BATCH_AsyrunJob: if (preq->rq_ind.rq_run.rq_destin) @@ -1323,8 +1312,6 @@ void free_br( - - static void freebr_manage( struct rq_manage *pmgr) diff --git a/src/server/req_runjob.c b/src/server/req_runjob.c index 812cf68da3..dfa352de9b 100644 --- a/src/server/req_runjob.c +++ b/src/server/req_runjob.c @@ -369,18 +369,7 @@ int req_runjob( if (preq->rq_type == PBS_BATCH_AsyrunJob) { - batch_request *new_preq = duplicate_request(preq, -1); - - if (new_preq == NULL) - { - sprintf(log_buf, "failed to duplicate batch request"); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - free_br(preq); - return(PBSE_MEM_MALLOC); - } - - reply_ack(new_preq); - free_br(new_preq); + reply_ack(preq); preq->rq_noreply = TRUE; enqueue_threadpool_request(check_and_run_job, preq, async_pool); } From b52eb793591daf24eb7d06a56fa6f7adeb8c137e Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Wed, 5 Oct 2016 12:19:31 -0600 Subject: [PATCH 006/312] Adjust torque build to shorten release candidate package names --- buildutils/torque-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index e3b95d32b5..500a0884e3 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -245,7 +245,7 @@ then make check | ./parse_cov_results.py fi -if [[ ${version} =~ ^[0-9]+(\.[0-9]+){2,3}(\.h[0-9]+)?$ ]] +if [[ ${version} =~ ^[0-9]+(\.[0-9]+){2,3}(\.(h|rc)[0-9]+)?$ ]] then package_version="${version}" release=1 From 7dedb1db4f51b37a4545f10182f66063acba61d1 Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Wed, 5 Oct 2016 12:33:40 -0600 Subject: [PATCH 007/312] Add beta to the list of shortened promotion levels --- buildutils/torque-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index 500a0884e3..1261b34d4c 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -245,7 +245,7 @@ then make check | ./parse_cov_results.py fi -if [[ ${version} =~ ^[0-9]+(\.[0-9]+){2,3}(\.(h|rc)[0-9]+)?$ ]] +if [[ ${version} =~ ^[0-9]+(\.[0-9]+){2,3}(\.(beta|h|rc)[0-9]+)?$ ]] then package_version="${version}" release=1 From 8b9ee51f6d9b457e574de5229ca91f5885785b4a Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Fri, 7 Oct 2016 14:20:25 -0600 Subject: [PATCH 008/312] Renovate version naming --- buildutils/torque-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index 1261b34d4c..219c8c1911 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -245,7 +245,7 @@ then make check | ./parse_cov_results.py fi -if [[ ${version} =~ ^[0-9]+(\.[0-9]+){2,3}(\.(beta|h|rc)[0-9]+)?$ ]] +if [[ ${version} =~ ^[0-9]+(\.[0-9]+){2,3}(\.?(a|b|r?c|h)\d+)?$ ]] then package_version="${version}" release=1 From 5acf53bdad830de32df68b165a9c339e6a24071a Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Fri, 7 Oct 2016 17:06:14 -0600 Subject: [PATCH 009/312] Oops, copied wrong digit range expression --- buildutils/torque-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index 219c8c1911..3991822b71 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -245,7 +245,7 @@ then make check | ./parse_cov_results.py fi -if [[ ${version} =~ ^[0-9]+(\.[0-9]+){2,3}(\.?(a|b|r?c|h)\d+)?$ ]] +if [[ ${version} =~ ^[0-9]+(\.[0-9]+){2,3}(\.?(a|b|r?c|h)[0-9]+)?$ ]] then package_version="${version}" release=1 From 034ed40df791abd56832808712b43b9311c68276 Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Mon, 10 Oct 2016 10:55:12 -0600 Subject: [PATCH 010/312] Support M.mrc1 as a temporary workaround --- buildutils/torque-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index 3991822b71..1de4201d5d 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -245,7 +245,7 @@ then make check | ./parse_cov_results.py fi -if [[ ${version} =~ ^[0-9]+(\.[0-9]+){2,3}(\.?(a|b|r?c|h)[0-9]+)?$ ]] +if [[ ${version} =~ ^[0-9]+(\.[0-9]+){1,3}(\.?(a|b|r?c|h)[0-9]+)?$ ]] then package_version="${version}" release=1 From 97865c26168888dfdf841bac49656e7400614738 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 10 Oct 2016 11:37:56 -0600 Subject: [PATCH 011/312] Attempt to get the adaptive spec to support cgroups. --- buildutils/torque.adaptive.spec.el7 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/buildutils/torque.adaptive.spec.el7 b/buildutils/torque.adaptive.spec.el7 index e9c0e311c7..242cd5b6b6 100644 --- a/buildutils/torque.adaptive.spec.el7 +++ b/buildutils/torque.adaptive.spec.el7 @@ -297,6 +297,7 @@ ### Features disabled by default %bcond_with blcr %bcond_with cpuset +%bcond_with cgroups %bcond_with libcpuset %bcond_with memacct %bcond_with munge @@ -312,6 +313,7 @@ ### Autoconf macro expansions %define ac_with_blcr --%{?with_blcr:en}%{!?with_blcr:dis}able-blcr %define ac_with_cpuset --%{?with_cpuset:en}%{!?with_cpuset:dis}able-cpuset +%define ac_with_cgroups --%{?with_cgroups:en}%{!?with_cgroups:dis}able-cgroups %define ac_with_drmaa --%{?with_drmaa:en}%{!?with_drmaa:dis}able-drmaa %define ac_with_munge --%{?with_munge:en}%{!?with_munge:dis}able-munge-auth %define ac_with_numa --%{?with_numa:en}%{!?with_numa:dis}able-numa-support @@ -327,6 +329,7 @@ %define breq_munge %{?with_munge:munge-devel} %define breq_pam %{?with_pam:pam-devel} %define breq_scp %{?with_scp:/usr/bin/scp} +%define req_cgroups %{?with_cgroups:%{rootbin}/lssubsys} # Missing: # libcpuset -> libcpuset.so.* @@ -429,6 +432,9 @@ Suite. %package %{common_sub} Group: Applications/System Summary: TORQUE Common Files +%if %{with cgroups} +Requires: %{req_cgroups} +%endif %description %{common_sub} Common files shared by TORQUE Server, Client, and MOM packages @@ -529,7 +535,9 @@ CXXFLAGS="-g3 -O0" --without-tcl \ %{ac_with_scp} %{ac_with_syslog} \ %{ac_with_munge} %{ac_with_pam} \ - %{ac_with_blcr} %{ac_with_cpuset} %{ac_with_spool} %{?acflags} + %{ac_with_blcr} %{ac_with_cpuset} \ + %{ac_with_cgroups} %{ac_with_spool} %{?acflags} + %{__make} clean %{__make} %{?_smp_mflags} %{?mflags} From 17e359b2b89ce2b449b41de99f4177e64c91a516 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 10 Oct 2016 15:15:02 -0600 Subject: [PATCH 012/312] TRQ-3454 - fix PAM authentication --- src/pam/Makefile.am | 2 +- src/test/pam_pbssimpleauth/Makefile.am | 2 ++ src/test/pam_pbssimpleauth/test_uut.c | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pam/Makefile.am b/src/pam/Makefile.am index e4e77bd6ff..99c59a0158 100644 --- a/src/pam/Makefile.am +++ b/src/pam/Makefile.am @@ -7,7 +7,7 @@ pammoddir=@pammoddir@ pammod_LTLIBRARIES = pam_pbssimpleauth.la -AM_CFLAGS = -DPBS_SERVER_HOME=\"$(PBS_SERVER_HOME)\" `xml2-config --cflags` +AM_CFLAGS = -DPBS_SERVER_HOME=\"$(PBS_SERVER_HOME)\" -DPBS_MOM `xml2-config --cflags` EXTRA_DIST = README.pam diff --git a/src/test/pam_pbssimpleauth/Makefile.am b/src/test/pam_pbssimpleauth/Makefile.am index c5c6d0803d..6641520581 100644 --- a/src/test/pam_pbssimpleauth/Makefile.am +++ b/src/test/pam_pbssimpleauth/Makefile.am @@ -1,3 +1,5 @@ include ../Makefile_pam.ut libuut_la_SOURCES = ${PROG_ROOT}/pam_pbssimpleauth.c + +AM_CFLAGS += -DPBS_MOM diff --git a/src/test/pam_pbssimpleauth/test_uut.c b/src/test/pam_pbssimpleauth/test_uut.c index 562601930a..38243ad4a6 100644 --- a/src/test/pam_pbssimpleauth/test_uut.c +++ b/src/test/pam_pbssimpleauth/test_uut.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include From 1614a648cece81f5cd07b7e8e439d01ddfe41c2e Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Mon, 10 Oct 2016 17:35:37 -0600 Subject: [PATCH 013/312] Workarounds for iris renormalization --- buildutils/torque-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index 1de4201d5d..9d10418296 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -245,7 +245,7 @@ then make check | ./parse_cov_results.py fi -if [[ ${version} =~ ^[0-9]+(\.[0-9]+){1,3}(\.?(a|b|r?c|h)[0-9]+)?$ ]] +if [[ ${version} =~ ^[0-9]+\.[0-9]+(\.?(a|b|r?c)[0-9]+|(\.[0-9]+)+(\.h[0-9]+)?)?$ ]] then package_version="${version}" release=1 From 2809a7f603791efc8fabd2d1b0e1a6d991953469 Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Mon, 10 Oct 2016 17:41:07 -0600 Subject: [PATCH 014/312] Regex fix --- buildutils/torque-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index 9d10418296..2aae11cf3e 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -245,7 +245,7 @@ then make check | ./parse_cov_results.py fi -if [[ ${version} =~ ^[0-9]+\.[0-9]+(\.?(a|b|r?c)[0-9]+|(\.[0-9]+)+(\.h[0-9]+)?)?$ ]] +if [[ ${version} =~ ^[0-9]+\.[0-9]+(\.?(a|b|r?c)[0-9]+|(\.[0-9]+)+(\.h[0-9]+)?)$ ]] then package_version="${version}" release=1 From a20521f108603c2ab87dae7b5b04ae0ae1fade23 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 13 Oct 2016 09:41:02 -0600 Subject: [PATCH 015/312] TRQ-3796 - simplify installation process --- Makefile.am | 9 +++++++++ buildutils/self-extract-head-sh.in | 15 ++++++++++++--- src/cmds/Makefile.am | 8 ++++++++ src/daemon_client/Makefile.am | 17 +++++++++++++++++ src/resmom/Makefile.am | 14 ++++++++++++++ src/server/Makefile.am | 7 +++++++ 6 files changed, 67 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 7a14bcca91..ef94b45397 100644 --- a/Makefile.am +++ b/Makefile.am @@ -174,6 +174,15 @@ packages: test $$d = gui && echo dummy > $$proot/regen-tclIndex; \ ( cd $$proot; tar cf - * ) | gzip > $$tarball && \ ( cd $$proot; tar cf - \ + `test -f ./usr/lib/systemd/system/pbs_server.service && echo ./usr/lib/systemd/system/pbs_server.service` \ + `test -f ./etc/init.d/pbs_server && echo ./etc/init.d/pbs_server` \ + `test -f ./usr/lib/systemd/system/pbs_mom.service && echo ./usr/lib/systemd/system/pbs_mom.service` \ + `test -f ./etc/init.d/pbs_mom && echo ./etc/init.d/pbs_mom` \ + `test -f ./usr/lib/systemd/system/trqauthd.service && echo ./usr/lib/systemd/system/trqauthd.service` \ + `test -f ./etc/init.d/trqauthd && echo ./etc/init.d/trqauthd` \ + `test -f ./etc/ld.so.conf.d/torque.conf && echo ./etc/ld.so.conf.d/torque.conf` \ + `test -f ./etc/profile.d/torque.sh && echo ./etc/profile.d/torque.sh` \ + `test -f ./etc/profile.d/torque.csh && echo ./etc/profile.d/torque.csh` \ `test -f ./post-install && echo ./post-install` \ `test -f ./regen-tclIndex && echo ./regen-tclIndex` \ `test -d .$(sbindir) && echo .$(sbindir)/*` \ diff --git a/buildutils/self-extract-head-sh.in b/buildutils/self-extract-head-sh.in index 9666f6f9b2..dcb826b48b 100644 --- a/buildutils/self-extract-head-sh.in +++ b/buildutils/self-extract-head-sh.in @@ -126,11 +126,11 @@ if test $install -gt 0;then # take the archive portion of this file and pipe it to tar sed -n $SKIP,\$p $0 | gzip -dc | (cd $destdir; tar x${vopt}f - ) - # finallized files should be owned by root, not the package-builder + # finalized files should be owned by root, not the package-builder if test $chownfiles -gt 0;then test $verbose -gt 0 && echo "setting file user and group ownerships" sed -n $SKIP,\$p $0 | gzip -dc | tar tf - | \ - while read line;do + while read line; do chown $user $destdir$line; chgrp $group $destdir$line case $line in *$sbindir/${program_prefix}pbs_iff${program_suffix}) @@ -143,6 +143,15 @@ if test $install -gt 0;then done fi + # see if we need to do an ldconfig + sed -n $SKIP,\$p $0 | gzip -dc | tar tf - | \ + (do_ldconfig=0; while read line; do + case $line in + */etc/ld.so.conf.d/torque.conf) + do_ldconfig=1;; + esac + done; test $do_ldconfig -gt 0 && /sbin/ldconfig) + # restore the original conf files if test $saveconfs -gt 0;then for a in $conffiles;do @@ -160,7 +169,7 @@ if test $install -gt 0;then echo "Skipping ${destdir}post-install" fi fi - + echo "Done." fi diff --git a/src/cmds/Makefile.am b/src/cmds/Makefile.am index 530ce7f5c5..ab4182888b 100644 --- a/src/cmds/Makefile.am +++ b/src/cmds/Makefile.am @@ -77,8 +77,16 @@ install-exec-hook: rm -f $(DESTDIR)$(bindir)/$(program_prefix)qnodes$(program_suffix)$(EXEEXT) ln -s $(program_prefix)pbsnodes$(program_suffix)$(EXEEXT) \ $(DESTDIR)$(bindir)/$(program_prefix)qnodes$(program_suffix)$(EXEEXT) + -$(install_sh) -d -m 755 $(DESTDIR)/etc/profile.d + test -f $(DESTDIR)/etc/profile.d/torque.sh || \ + echo "PATH=$(bindir):$(sbindir):\$$PATH" > $(DESTDIR)/etc/profile.d/torque.sh + test -f $(DESTDIR)/etc/profile.d/torque.csh || \ + echo "set path = ($(bindir) $(sbindir) \$$path)" > $(DESTDIR)/etc/profile.d/torque.csh + chmod 644 $(DESTDIR)/etc/profile.d/torque.sh $(DESTDIR)/etc/profile.d/torque.csh uninstall-hook: rm -f $(DESTDIR)$(PBS_DEFAULT_FILE) rm -f $(DESTDIR)$(bindir)/$(program_prefix)qnodes$(program_suffix)$(EXEEXT) + -rm -f $(DESTDIR)/etc/profile.d/torque.sh + -rm -f $(DESTDIR)/etc/profile.d/torque.csh diff --git a/src/daemon_client/Makefile.am b/src/daemon_client/Makefile.am index 6fb5b12153..a72edbe7ec 100644 --- a/src/daemon_client/Makefile.am +++ b/src/daemon_client/Makefile.am @@ -11,3 +11,20 @@ sbin_PROGRAMS = trqauthd trqauthd_SOURCES = trq_auth_daemon.c trq_main.c trqauthd_LDFLAGS = -L../lib/Libpbs/.libs -ltorque +install-exec-hook: + systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1 && \ + echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service" && \ + $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || \ + (echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd" && \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd) + -$(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d + test -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || \ + echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf && chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf && \ + echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig + +uninstall-hook: + -rm -f $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || \ + rm -f $(DESTDIR)/etc/init.d/trqauthd + -rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf + echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig + diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index 2084ba9079..632a1590a1 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -72,8 +72,22 @@ install-exec-hook: rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qnoded$(program_suffix)$(EXEEXT) ln -s $(program_prefix)pbs_mom$(program_suffix)$(EXEEXT) \ $(DESTDIR)$(sbindir)/$(program_prefix)qnoded$(program_suffix)$(EXEEXT) + systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1 && \ + echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service" && \ + $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service || \ + (echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom" && \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom) + -$(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d + test -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || \ + echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf && \ + chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf && \ + echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig uninstall-hook: rm -f $(DESTDIR)$(PBS_ENVIRON) rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qnoded$(program_suffix)$(EXEEXT) + -rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service || \ + rm -f $(DESTDIR)/etc/init.d/pbs_mom + -rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf + echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig diff --git a/src/server/Makefile.am b/src/server/Makefile.am index 7fe5b0975c..93232ec8c8 100644 --- a/src/server/Makefile.am +++ b/src/server/Makefile.am @@ -54,8 +54,15 @@ install-exec-hook: rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qserverd$(program_suffix)$(EXEEXT) ln -s $(program_prefix)pbs_server$(program_suffix)$(EXEEXT) \ $(DESTDIR)$(sbindir)/$(program_prefix)qserverd$(program_suffix)$(EXEEXT) + systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1 && \ + echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service" && \ + $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service || \ + (echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server" && \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server) uninstall-hook: rm -f $(DESTDIR)$(PBS_ENVIRON) $(DESTDIR)$(PBS_SERVER_HOME)/server_priv/nodes rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qserverd$(program_suffix)$(EXEEXT) + -rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_server.service || \ + rm -f $(DESTDIR)/etc/init.d/pbs_server From e34370321386a8c8581b9831c4b035e6825fffd7 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 13 Oct 2016 12:08:46 -0600 Subject: [PATCH 016/312] Replace most uses of the customer AVL tree with a map. This is because we found a strange bug in the AVL tree (it gets out of order) on SLES12. Also fix a couple of potential places for memory leaks. --- configure.ac | 1 + src/include/Makefile.am | 2 +- src/include/authorized_hosts.hpp | 51 ++++ src/include/svrfunc.h | 4 +- src/lib/Libutils/Makefile.am | 2 +- src/lib/Libutils/authorized_hosts.cpp | 264 ++++++++++++++++++ src/resmom/mom_comm.c | 34 +-- src/resmom/mom_main.c | 38 ++- src/resmom/mom_process_request.c | 8 +- src/resmom/mom_server.c | 43 +-- src/resmom/parse_config.c | 5 +- src/server/node_func.c | 8 +- src/server/node_manager.c | 163 ++++------- src/server/pbsd_main.c | 3 +- src/server/receive_mom_communication.c | 10 +- src/server/req_runjob.c | 44 ++- src/server/svr_mail.c | 1 + src/test/Makefile.am | 28 +- src/test/authorized_hosts/Makefile.am | 4 + src/test/authorized_hosts/scaffolding.c | 42 +++ src/test/authorized_hosts/test_uut.c | 120 ++++++++ src/test/mom_comm/scaffolding.c | 11 + src/test/mom_main/scaffolding.c | 12 + src/test/mom_process_request/scaffolding.c | 10 + src/test/mom_server/scaffolding.c | 14 + src/test/node_func/scaffolding.c | 12 +- src/test/node_manager/scaffolding.c | 9 + src/test/parse_config/scaffolding.c | 7 + src/test/pbsd_main/scaffolding.c | 3 + .../receive_mom_communication/scaffolding.c | 9 + src/test/req_runjob/scaffolding.c | 2 +- 31 files changed, 720 insertions(+), 244 deletions(-) create mode 100644 src/include/authorized_hosts.hpp create mode 100644 src/lib/Libutils/authorized_hosts.cpp create mode 100644 src/test/authorized_hosts/Makefile.am create mode 100644 src/test/authorized_hosts/scaffolding.c create mode 100644 src/test/authorized_hosts/test_uut.c diff --git a/configure.ac b/configure.ac index bcc1f20bec..c57e714c7e 100644 --- a/configure.ac +++ b/configure.ac @@ -429,6 +429,7 @@ m4_ifdef([AM_SILENT_RULES],[ src/test/site_mom_chu/Makefile src/test/site_mom_ckp/Makefile src/test/site_mom_jst/Makefile + src/test/authorized_hosts/Makefile src/test/u_MXML/Makefile src/test/u_groups/Makefile src/test/u_hash_map_structs/Makefile diff --git a/src/include/Makefile.am b/src/include/Makefile.am index 44db98d680..4df40c7a96 100644 --- a/src/include/Makefile.am +++ b/src/include/Makefile.am @@ -28,7 +28,7 @@ noinst_HEADERS = acct.h array.h assertions.h attribute.h \ job_recovery.h allocation.hpp attr_req_info.hpp acl_special.hpp restricted_host.hpp \ pbs_helper.h mail_throttler.hpp lib_ifl.h runjob_help.hpp pmix_tracker.hpp \ pmix_operation.hpp job_host_data.hpp policy_values.h plugin_internal.h json/json.h \ - json/json-forwards.h + json/json-forwards.h authorized_hosts.hpp BUILT_SOURCES = site_job_attr_def.h site_job_attr_enum.h \ site_qmgr_node_print.h site_qmgr_que_print.h \ diff --git a/src/include/authorized_hosts.hpp b/src/include/authorized_hosts.hpp new file mode 100644 index 0000000000..0e14dbb36a --- /dev/null +++ b/src/include/authorized_hosts.hpp @@ -0,0 +1,51 @@ + +#ifndef __AUTHORIZED_HOSTS_HPP__ +#define __AUTHORIZED_HOSTS_HPP__ +#include +#include + +#include "pbs_nodes.h" +#include + +class authorized_host_info + { + friend class authorized_hosts; + + std::string hostname; + std::set ports; + + public: + authorized_host_info() : hostname(), ports() {} + authorized_host_info(const std::string &host, unsigned short port) : hostname(host) + { + ports.insert(port); + } + + authorized_host_info( + const authorized_host_info &other) : hostname(other.hostname), ports(other.ports) {} + }; + + +class authorized_hosts + { + pthread_mutex_t auth_mutex; + // Map key is the ip address, value is the host information + std::map > auth_map; + + public: + authorized_hosts(); + + void add_authorized_address(unsigned long addr, unsigned short port, const std::string &host); + void clear(); + + bool is_authorized(unsigned long addr); + bool is_authorized(unsigned long addr, unsigned short port); + pbsnode *get_authorized_node(unsigned long addr); + pbsnode *get_authorized_node(unsigned long addr, unsigned short port); + void list_authorized_hosts(std::string &output); + bool remove_address(unsigned long addr, unsigned short port); + }; + +extern authorized_hosts auth_hosts; + +#endif diff --git a/src/include/svrfunc.h b/src/include/svrfunc.h index 362332e0b1..5ad97f45d1 100644 --- a/src/include/svrfunc.h +++ b/src/include/svrfunc.h @@ -5,6 +5,8 @@ #include "batch_request.h" #include "get_path_jobdata.h" +#include + /* * misc server function prototypes */ @@ -41,7 +43,7 @@ int is_svr_attr_set(int); int set_svr_attr(int, void *); #ifdef PBS_JOB_H -extern int set_nodes(job *, char *, int, char **, char **, char *, char *); +extern int set_nodes(job *, const char *, int, std::string &, std::string &, char *, char *); extern void free_nodes(job *, const char *spec = NULL); #endif /* PBS_JOB_H */ diff --git a/src/lib/Libutils/Makefile.am b/src/lib/Libutils/Makefile.am index 0f22455e36..c56e858e70 100644 --- a/src/lib/Libutils/Makefile.am +++ b/src/lib/Libutils/Makefile.am @@ -14,5 +14,5 @@ libutils_a_SOURCES = u_groups.c u_tree.c u_mu.c u_MXML.c u_xml.c \ u_constants.c u_mutex_mgr.cpp \ u_misc.c u_putenv.c u_wrapper.c u_timer.cpp \ machine.cpp numa_chip.cpp numa_core.cpp numa_pci_device.cpp \ - numa_socket.cpp allocation.cpp jsoncpp.cpp + numa_socket.cpp allocation.cpp jsoncpp.cpp authorized_hosts.cpp diff --git a/src/lib/Libutils/authorized_hosts.cpp b/src/lib/Libutils/authorized_hosts.cpp new file mode 100644 index 0000000000..868b275c54 --- /dev/null +++ b/src/lib/Libutils/authorized_hosts.cpp @@ -0,0 +1,264 @@ + +#include + +#include "authorized_hosts.hpp" +#include "log.h" + + + +authorized_hosts::authorized_hosts() : auth_map() + + { + pthread_mutex_init(&this->auth_mutex, NULL); + } // END constructor + + + +/* + * add_authorized_address() + * + * Adds an authorized host with it's information + * @param addr - the address of the host + * @param port - the port of the host + * @param host - the hostname of the host + */ + +void authorized_hosts::add_authorized_address( + + unsigned long addr, + unsigned short port, + const std::string &host) + + { + pthread_mutex_lock(&this->auth_mutex); + + if (this->auth_map.find(addr) == this->auth_map.end()) + { + std::map port_host_map; + port_host_map[port] = host; + this->auth_map[addr] = port_host_map; + } + else + this->auth_map[addr][port] = host; + pthread_mutex_unlock(&this->auth_mutex); + } // END add_authorized_address() + + + +/* + * clear() + * + * Clears the current authorized host information + */ + +void authorized_hosts::clear() + + { + pthread_mutex_lock(&this->auth_mutex); + this->auth_map.clear(); + pthread_mutex_unlock(&this->auth_mutex); + } + + + +/* + * is_authorized() + * + * Determines if the address in question is authorized. + * @param addr - the addr that we want to know if is authorized or not + */ + +bool authorized_hosts::is_authorized( + + unsigned long addr) + + { + bool legit = false; + + pthread_mutex_lock(&this->auth_mutex); + if (this->auth_map.find(addr) != this->auth_map.end()) + legit = true; + pthread_mutex_unlock(&this->auth_mutex); + + return(legit); + } // END is_authorized() + + + +/* + * is_authorized() + * + * Determines if the address and port in question is authorized. + * @param addr - the addr that we want to know if is authorized or not + * @param port - the port that we want to know if is authorized or not + */ + +bool authorized_hosts::is_authorized( + + unsigned long addr, + unsigned short port) + + { + bool legit = false; + + pthread_mutex_lock(&this->auth_mutex); + if (this->auth_map.find(addr) != this->auth_map.end()) + { + if (this->auth_map[addr].find(port) != this->auth_map[addr].end()) + legit = true; + } + pthread_mutex_unlock(&this->auth_mutex); + + return(legit); + } // END is_authorized() + + + +/* + * get_authorized_node() + * + * Retrieves the node associated with the addr, if there is an authorized node associated with + * this address. + * + * @param addr - the address in question + * @return the node associated with the address + */ + +pbsnode *authorized_hosts::get_authorized_node( + + unsigned long addr) + + { + std::string hostname; + pbsnode *pnode = NULL; + + pthread_mutex_lock(&this->auth_mutex); + if (this->auth_map.find(addr) != this->auth_map.end()) + hostname = this->auth_map[addr].begin()->second; + pthread_mutex_unlock(&this->auth_mutex); + + if (hostname.size() != 0) + pnode = find_nodebyname(hostname.c_str()); + + return(pnode); + } // END get_authorized_node() + + + +/* + * get_authorized_node() + * + * Retrieves the node associated with the addr and port, if there is an authorized node associated + * with this address. + * + * @param addr - the address in question + * @param port - the port in question + * @return the node associated with the address + */ + +pbsnode *authorized_hosts::get_authorized_node( + + unsigned long addr, + unsigned short port) + + { + std::string hostname; + pbsnode *pnode = NULL; + + pthread_mutex_lock(&this->auth_mutex); + if (this->auth_map.find(addr) != this->auth_map.end()) + { + if (this->auth_map[addr].find(port) != this->auth_map[addr].end()) + hostname = this->auth_map[addr][port]; + } + pthread_mutex_unlock(&this->auth_mutex); + + if (hostname.size() != 0) + pnode = find_nodebyname(hostname.c_str()); + + return(pnode); + } // END get_authorized_node() + + + +/* + * list_authorized_hosts() + * + * Outputs the authorized hosts into a list + */ + +void authorized_hosts::list_authorized_hosts( + + std::string &output) + + { + char buf[MAXLINE]; + + output.clear(); + + pthread_mutex_lock(&this->auth_mutex); + + for (std::map >::iterator it = this->auth_map.begin(); + it != this->auth_map.end(); + it++) + { + for (std::map::iterator map_it = it->second.begin(); + map_it != it->second.end(); + map_it++) + { + sprintf(buf, "%ld.%ld.%ld.%ld:%d", + (it->first & 0xFF000000) >> 24, + (it->first & 0x00FF0000) >> 16, + (it->first & 0x0000FF00) >> 8, + (it->first & 0x000000FF), + map_it->first); + + if (output.size() != 0) + output += ","; + + output += buf; + } + } + pthread_mutex_unlock(&this->auth_mutex); + } // END list_authorized_hosts() + + + +/* + * remove_address() + * Removes the address and port from the authorized hosts + * + * @param host - the host that should be removed + * @param port - the accompanying port + * @return true if the node was removed, false if it wasn't present and therefore wasn't removed + */ + +bool authorized_hosts::remove_address( + + unsigned long addr, + unsigned short port) + + { + bool removed = false; + + pthread_mutex_lock(&this->auth_mutex); + std::map >::iterator it = this->auth_map.find(addr); + + std::map::iterator map_it = it->second.find(port); + if (map_it != it->second.end()) + { + removed = true; + it->second.erase(map_it); + + // If there are no authorized ports left, remove this address completely + if (it->second.size() == 0) + { + this->auth_map.erase(it); + } + } + + pthread_mutex_unlock(&this->auth_mutex); + + return(removed); + } // END remove_address() + diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index b7a294e72d..fb61d6a48b 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -122,7 +122,7 @@ #include "mom_comm.h" #include "mcom.h" #include "svrfunc.h" -#include "u_tree.h" +#include "authorized_hosts.hpp" #include "utils.h" #include "../lib/Libnet/lib_net.h" /* get_hostaddr_hostent_af */ #include "mom_server.h" @@ -159,7 +159,6 @@ extern tlist_head svr_newjobs; extern tlist_head mom_polljobs; /* must have resource limits polled */ extern int termin_child; extern time_t time_now; -extern AvlTree okclients; extern int port_care; extern char *path_prologp; extern char *path_prologuserp; @@ -6040,35 +6039,20 @@ void im_request( log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, __func__, log_buffer); } - - if (AVL_is_in_tree_no_port_compare(ipaddr, 0, okclients) == 0 ) + + if (auth_hosts.is_authorized(ipaddr) == false) { - long max_len = 1024; - long final_len = 0; - char *tmp_line = (char *)calloc(1, max_len + 1); + std::string list; close_conn(chan->sock, FALSE); svr_conn[chan->sock].cn_stay_open = FALSE; chan->sock = -1; - if (tmp_line != NULL) - ret = AVL_list(okclients, &tmp_line, &final_len, &max_len); - else - ret = -1; - if (ret == 0) - { - snprintf(log_buffer, LOCAL_LOG_BUF_SIZE, - "bad connect from %s - unauthorized (okclients: %s)", - netaddr(pSockAddr), - tmp_line); - } - else - snprintf(log_buffer, LOCAL_LOG_BUF_SIZE, - "bad connect from %s - unauthorized (could not get ok clients %d)", - netaddr(pSockAddr), - ret); + auth_hosts.list_authorized_hosts(list); + snprintf(log_buffer, LOCAL_LOG_BUF_SIZE, + "bad connect from %s - unauthorized (okclients: %s)", + netaddr(pSockAddr), + list.c_str()); - if (tmp_line != NULL) - free(tmp_line); log_err(-1, __func__, log_buffer); goto im_req_finish; diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index 2ec8e2d653..b52e453a3e 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -80,7 +80,7 @@ #include "dis.h" #include "csv.h" #include "utils.h" -#include "u_tree.h" +#include "authorized_hosts.hpp" #ifdef PENABLE_LINUX26_CPUSETS #include "pbs_cpuset.h" #include "node_internals.hpp" @@ -224,6 +224,7 @@ extern struct var_table vtable; /* see start_exec.c */ time_t last_log_check; +authorized_hosts auth_hosts; std::vector exiting_job_list; std::vector things_to_resend; @@ -360,7 +361,6 @@ int port_care = FALSE; /* configured without priv ports, don't care about them uid_t uid = 0; /* uid we are running with */ unsigned int alarm_time = 10; /* time before alarm */ -extern AvlTree okclients; /* accept connections from */ u_long localaddr = 0; int cphosts_num = 0; @@ -1948,22 +1948,12 @@ void add_diag_okclient_list( std::stringstream &output) { - long max_len = 1024; - long final_len = 0; - char *tmp_line = (char *)calloc(1, max_len + 1); - - if (tmp_line != NULL) - { - int ret = AVL_list(okclients, &tmp_line, &final_len, &max_len); + std::string list; + + auth_hosts.list_authorized_hosts(list); - output << "Trusted Client List: " << tmp_line << ": " << ret << "\n"; - free(tmp_line); - } - else - { - output << "Trusted Client Could not be retrieved\n"; - } - } + output << "Trusted Client List: " << list << "\n"; + } // END add_diag_okclient_list() void add_diag_copy_command( @@ -2907,7 +2897,7 @@ int rm_request( } if (((port_care != FALSE) && (port >= IPPORT_RESERVED)) || - (AVL_is_in_tree_no_port_compare(ipadd, 0, okclients) == 0 )) + (auth_hosts.is_authorized(ipadd) == false)) { if (bad_restrict(ipadd)) { @@ -3441,7 +3431,7 @@ void *tcp_request( log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); } - if (AVL_is_in_tree_no_port_compare(ipadd, 0, okclients) == 0) + if (auth_hosts.is_authorized(ipadd) == false) { sprintf(log_buffer, "bad connect from %s", address); log_err(-1, __func__, log_buffer); @@ -4560,7 +4550,7 @@ bool verify_mom_hierarchy() { unsigned short rm_port = ntohs(nc.sock_addr.sin_port); unsigned long ipaddr = ntohl(nc.sock_addr.sin_addr.s_addr); - okclients = AVL_insert(ipaddr, rm_port, NULL, okclients); + auth_hosts.add_authorized_address(ipaddr, rm_port, ""); } legitimate_hierarchy = true; @@ -5469,7 +5459,7 @@ int setup_program_environment(void) localaddr = ntohl(inet_addr("127.0.0.1")); - okclients = AVL_insert(localaddr, 0, NULL, okclients); + auth_hosts.add_authorized_address(localaddr, 0, ""); addclient(mom_host); @@ -7554,6 +7544,12 @@ void get_mom_job_dir_sticky_config( } /* END get_mom_job_dir_sticky_config */ +// This is a stub that shouldn't ever be used +pbsnode *find_nodebyname(const char *nodename) + { + return(NULL); + } + /* END mom_main.c */ diff --git a/src/resmom/mom_process_request.c b/src/resmom/mom_process_request.c index 687981efb4..46c0909c1a 100644 --- a/src/resmom/mom_process_request.c +++ b/src/resmom/mom_process_request.c @@ -36,7 +36,7 @@ #include "svrfunc.h" #include "pbs_proto.h" #include "csv.h" -#include "u_tree.h" +#include "authorized_hosts.hpp" #include "threadpool.h" #include "dis.h" #include "mom_job_func.h" @@ -72,8 +72,6 @@ extern char *msg_err_noqueue; extern char *msg_err_malloc; extern char *msg_request; -extern AvlTree okclients; - extern int LOGLEVEL; /* private functions local to this file */ @@ -226,8 +224,6 @@ void *mom_process_request( /* is the request from a host acceptable to the server */ { - /*extern tree *okclients; */ - extern void mom_server_update_receive_time_by_ip(u_long ipaddr, const char *cmd); /* check connecting host against allowed list of ok clients */ @@ -254,7 +250,7 @@ void *mom_process_request( return NULL; } - if (!AVL_is_in_tree_no_port_compare(svr_conn[chan->sock].cn_addr, 0, okclients)) + if (auth_hosts.is_authorized(svr_conn[chan->sock].cn_addr) == false) { sprintf(log_buffer, "request type %s from host %s rejected (host not authorized)", reqtype_to_txt(request->rq_type), diff --git a/src/resmom/mom_server.c b/src/resmom/mom_server.c index d2f1fc5659..88d7542517 100644 --- a/src/resmom/mom_server.c +++ b/src/resmom/mom_server.c @@ -229,7 +229,7 @@ #include "server_limits.h" #include "pbs_job.h" #include "utils.h" -#include "u_tree.h" +#include "authorized_hosts.hpp" #include "mom_hierarchy.h" #include "mom_server.h" #include "mom_comm.h" @@ -297,7 +297,6 @@ extern long system_ncpus; extern int alarm_time; /* time before alarm */ extern time_t time_now; extern int verbositylevel; -extern AvlTree okclients; extern tlist_head mom_polljobs; extern char mom_alias[]; extern int updates_waiting_to_send; @@ -583,7 +582,7 @@ int mom_server_add( if (ipaddr != 0) { - okclients = AVL_insert(ipaddr, 0, NULL, okclients); + auth_hosts.add_authorized_address(ipaddr, 0, ""); } } @@ -2151,12 +2150,6 @@ void mom_server_update_receive_time_by_ip( /** ** Modified by Tom Proett for PBS. */ - -/*tree *okclients = NULL;*/ /* tree of ip addrs */ -AvlTree okclients = NULL; - - - /** * mom_server_valid_message_source * @@ -2236,7 +2229,7 @@ mom_server *mom_server_valid_message_source( if (ipaddr == server_ip) { - okclients = AVL_insert(ipaddr, 0, NULL, okclients); + auth_hosts.add_authorized_address(ipaddr, 0, ""); return(pms); } @@ -2296,7 +2289,7 @@ int process_host_name( } /* add to acceptable host tree */ - okclients = AVL_insert(ipaddr, rm_port, NULL, okclients); + auth_hosts.add_authorized_address(ipaddr, rm_port, ""); } else { @@ -2413,7 +2406,7 @@ void sort_paths() void reset_okclients() { - okclients = AVL_clear_tree(okclients); + auth_hosts.clear(); // re-add each server for (int sindex = 0;sindex < PBS_MAXSERVER;sindex++) @@ -2440,14 +2433,14 @@ void reset_okclients() if (ipaddr != 0) { - okclients = AVL_insert(ipaddr, 0, NULL, okclients); + auth_hosts.add_authorized_address(ipaddr, 0, ""); } } } } // add localhost - okclients = AVL_insert(localaddr, 0, NULL, okclients); + auth_hosts.add_authorized_address(localaddr, 0, ""); // BMD: add the node's ip address @@ -2467,10 +2460,7 @@ int read_cluster_addresses( int path_complete = FALSE; int something_added; char *str; - char *okclients_list; std::string hierarchy_file = "/n"; - long list_size; - long list_len = 0; if (mh != NULL) free_mom_hierarchy(mh); @@ -2563,17 +2553,12 @@ int read_cluster_addresses( sort_paths(); /* log the hierrarchy */ - list_size = MAXLINE * 2; - if ((okclients_list = (char *)calloc(1, list_size)) != NULL) - { - AVL_list(okclients, &okclients_list, &list_len, &list_size); - snprintf(log_buffer, sizeof(log_buffer), - "Successfully received the mom hierarchy file. My okclients list is '%s', and the hierarchy file is '%s'", - okclients_list, hierarchy_file.c_str()); - log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_NODE, __func__, log_buffer); - - free(okclients_list); - } + std::string list; + auth_hosts.list_authorized_hosts(list); + snprintf(log_buffer, sizeof(log_buffer), + "Successfully received the mom hierarchy file. My okclients list is '%s', and the hierarchy file is '%s'", + list.c_str(), hierarchy_file.c_str()); + log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_NODE, __func__, log_buffer); /* tell the mom to go ahead and send an update to pbs_server */ first_update_time = 0; @@ -2642,7 +2627,7 @@ void mom_is_request( { ipaddr = ntohl(pAddr->sin_addr.s_addr); - if (AVL_is_in_tree_no_port_compare(ipaddr, 0, okclients) == 0) + if (auth_hosts.is_authorized(ipaddr) == false) { if (err_msg) { diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index 05d98385fa..d7582a9a35 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -97,7 +97,7 @@ #include "mom_config.h" #include "dis.h" #include "mom_func.h" -#include "u_tree.h" +#include "authorized_hosts.hpp" #include "csv.h" #include "json/json.h" @@ -201,7 +201,6 @@ extern char PBSNodeMsgBuf[MAXLINE]; extern long MaxConnectTimeout; extern char *path_log; extern tlist_head mom_varattrs; /* variable attributes */ -extern AvlTree okclients; /* accept connections from */ extern int mom_server_count; extern time_t time_now; extern int internal_state; @@ -1424,7 +1423,7 @@ u_long addclient( ipaddr = ntohl(saddr.s_addr); - okclients = AVL_insert(ipaddr, 0, NULL, okclients); + auth_hosts.add_authorized_address(ipaddr, 0, ""); return(ipaddr); } /* END addclient() */ diff --git a/src/server/node_func.c b/src/server/node_func.c index d95a38fe96..be3a035ee6 100644 --- a/src/server/node_func.c +++ b/src/server/node_func.c @@ -65,6 +65,7 @@ #include "mom_hierarchy_handler.h" #include "runjob_help.hpp" #include "policy_values.h" +#include "authorized_hosts.hpp" #if !defined(H_ERRNO_DECLARED) && !defined(_AIX) /*extern int h_errno;*/ @@ -84,7 +85,6 @@ extern char *path_nodepowerstate; extern char *path_nodenote; extern int LOGLEVEL; extern attribute_def node_attr_def[]; /* node attributes defs */ -extern AvlTree ipaddrs; job *get_job_from_job_usage_info(job_usage_info *jui, struct pbsnode *pnode); @@ -887,7 +887,7 @@ void effective_node_delete( for (unsigned int i = 0; i < pnode->nd_addrs.size(); i++) { /* del node's IP addresses from tree */ - ipaddrs = AVL_delete_node(pnode->nd_addrs[i], pnode->nd_mom_port, ipaddrs); + auth_hosts.remove_address(pnode->nd_addrs[i], pnode->nd_mom_port); } delete pnode; @@ -1645,7 +1645,7 @@ static int finalize_create_pbs_node(char *pname, /* node name w/o any :ts } addr = pul[i]; - ipaddrs = AVL_insert(addr, pnode->nd_mom_port, pnode, ipaddrs); + auth_hosts.add_authorized_address(addr, pnode->nd_mom_port, pnode->get_name()); } /* END for (i) */ if ((rc = setup_node_boards(pnode, pul)) != PBSE_NONE) @@ -3173,7 +3173,7 @@ int create_partial_pbs_node( } insert_node(&allnodes,pnode); - AVL_insert(addr, pnode->nd_mom_port, pnode, ipaddrs); + auth_hosts.add_authorized_address(addr, pnode->nd_mom_port, pnode->get_name()); svr_totnodes++; recompute_ntype_cnts(); diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 1cd2ab6182..56b03c5557 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -141,6 +141,7 @@ #include "runjob_help.hpp" #include "plugin_internal.h" #include "json/json.h" +#include "authorized_hosts.hpp" #define IS_VALID_STR(STR) (((STR) != NULL) && ((STR)[0] != '\0')) @@ -234,13 +235,11 @@ struct pbsnode *tfind_addr( char *job_momname) { - struct pbsnode *pn = AVL_find(key,port,ipaddrs); + pbsnode *pn = auth_hosts.get_authorized_node(key, port); if (pn == NULL) return(NULL); - pn->lock_node(__func__, "pn", LOGLEVEL); - if ((pn->num_node_boards == 0) || (job_momname == NULL)) return(pn); @@ -1375,7 +1374,7 @@ void stream_eof( if (addr != 0) { - np = AVL_find(addr, port, ipaddrs); + np = auth_hosts.get_authorized_node(addr, port); } if (np == NULL) @@ -2593,8 +2592,8 @@ int save_node_for_adding( void set_first_node_name( - char *spec_param, /* I */ - char *first_node_name) /* O */ + const char *spec_param, /* I */ + char *first_node_name) /* O */ { int i; @@ -3197,7 +3196,7 @@ bool process_as_node_list( int node_spec( - char *spec_param, /* I */ + const char *spec_param, /* I */ int early, /* I (boolean) */ int exactmatch, /* I (boolean) - NOT USED */ char *ProcBMStr, /* I */ @@ -4292,94 +4291,56 @@ int place_subnodes_in_hostlist( /* - * takes a struct howl and translates it to a string that will + * translate_howl_to_string() + * + * Takes a list of howl objects and translates it to a string that will * become a job pbs_attribute (exec_hosts, exec_gpus, exec_ports) - * NOTE: frees list (the struct howl) + * + * @param hlist - a std list of the howl objects + * @param EMsg - output any error message here + * @param NCount - the count of nodes in the string + * @param str - output the populated string here + * @param portstr - output the populated port string here + * @param port - if true, populate the port string + * @retunrn PBSE_NONE on success of PBSE_* on error */ int translate_howl_to_string( std::list &hlist, char *EMsg, - int *NCount, - char **str_ptr, - char **portstr_ptr, - int port) + int &NCount, + std::string &str, + std::string *portstr, + bool port) { - size_t len = 1; - int count = 1; - char *str; - char *end; - char *portlist = NULL; - char *endport; - - for (std::list::iterator it = hlist.begin(); it != hlist.end(); it++) - { - len += it->hostname.size() + 8; - count++; - } - - if ((str = (char *)calloc(1, len + 1)) == NULL) - { - log_err(ENOMEM, __func__, "Cannot allocate memory!"); - - if (EMsg != NULL) - sprintf(EMsg,"no nodes can be allocated to job"); - - return(PBSE_RESCUNAV); - } - - *str = '\0'; - - if (port == TRUE) - { - /* port list will have a string of sister port addresses */ - if ((portlist = (char *)calloc(1, (count * PBS_MAXPORTNUM) + count)) == NULL) - { - log_err(ENOMEM, __func__, "Cannot allocate memory!"); - - if (EMsg != NULL) - sprintf(EMsg,"no nodes can be allocated to job"); - - free(str); - - return(PBSE_RESCUNAV); - } - - *portlist = '\0'; - } + char buf[MAXLINE]; /* now copy in name+name+... */ - *NCount = 0; + NCount = 0; - end = str; - endport = portlist; for (std::list::iterator it = hlist.begin(); it != hlist.end(); it++) { - (*NCount)++; + NCount++; - sprintf(end, "%s/%d+", - it->hostname.c_str(), - it->index); + if (str.size() != 0) + str += "+"; + + str += it->hostname + "/"; - end += strlen(end); + sprintf(buf, "%d", it->index); + str += buf; - if (port == TRUE) + if ((port == true) && + (portstr != NULL)) { - sprintf(endport, "%d+", it->port); - endport += strlen(endport); - } - } - - /* strip trailing '+' and assign pointers */ - str[strlen(str) - 1] = '\0'; - *str_ptr = str; + if (portstr->size() != 0) + portstr->append("+"); - if (port == TRUE) - { - portlist[strlen(portlist) - 1] = '\0'; - *portstr_ptr = portlist; + sprintf(buf, "%d", it->port); + portstr->append(buf); + } } return(PBSE_NONE); @@ -4555,7 +4516,7 @@ int build_hostlist_nodes_req( job *pjob, /* M */ char *EMsg, /* O */ - char *spec, /* I */ + const char *spec, /* I */ short newstate, /* I */ std::vector &host_info, /* O */ std::list &gpu_list, /* O */ @@ -4905,18 +4866,17 @@ int locate_resource_request_20_nodes( int set_nodes( - job *pjob, /* I */ - char *spec, /* I */ - int procs, /* I */ - char **rtnlist, /* O */ - char **rtnportlist, /* O */ - char *FailHost, /* O (optional,minsize=1024) */ - char *EMsg) /* O (optional,minsize=1024) */ + job *pjob, /* I */ + const char *spec, /* I */ + int procs, /* I */ + std::string &node_list, /* O */ + std::string &rtnportlist, /* O */ + char *FailHost, /* O (optional,minsize=1024) */ + char *EMsg) /* O (optional,minsize=1024) */ { FUNCTION_TIMER std::vector host_info; - std::string exec_hosts; std::stringstream exec_ports; std::list naji_list; std::list gpu_list; @@ -4928,8 +4888,8 @@ int set_nodes( short newstate; char *login_prop = NULL; - char *gpu_str = NULL; - char *mic_str = NULL; + std::string gpu_str; + std::string mic_str; char ProcBMStr[MAX_BM]; char log_buf[LOCAL_LOG_BUF_SIZE]; alps_req_data *ard_array = NULL; @@ -5069,7 +5029,7 @@ int set_nodes( pjob->ji_qs.ji_svrflags |= JOB_SVFLG_HasNodes; /* indicate has nodes */ /* build list of allocated nodes, gpus, and ports */ - rc = translate_job_reservation_info_to_string(host_info, &NCount, exec_hosts, &exec_ports); + rc = translate_job_reservation_info_to_string(host_info, &NCount, node_list, &exec_ports); if (rc != PBSE_NONE) { free_nodes(pjob, spec); @@ -5077,29 +5037,24 @@ int set_nodes( return(rc); } - *rtnlist = strdup(exec_hosts.c_str()); - *rtnportlist = strdup(exec_ports.str().c_str()); + rtnportlist = exec_ports.str(); // JOB_TYPE_normal means no component from the Cray will be used if ((cray_enabled == true) && (job_type != JOB_TYPE_normal)) { - char *plus = strchr(*rtnlist, '+'); + size_t pos = node_list.find('+'); /* only do this if there's more than one host in the host list */ - if (plus != NULL) + if (pos != std::string::npos) { - char *to_free = *rtnlist; - - *plus = '\0'; - *rtnlist = strdup(plus + 1); - free(to_free); + node_list = node_list.substr(pos + 1); } } if (mic_list.size() != 0) { - if ((rc = translate_howl_to_string(mic_list, EMsg, &NCount, &mic_str, NULL, FALSE)) != PBSE_NONE) + if ((rc = translate_howl_to_string(mic_list, EMsg, NCount, mic_str, NULL, false)) != PBSE_NONE) { free_nodes(pjob, spec); return(rc); @@ -5112,15 +5067,13 @@ int set_nodes( &pjob->ji_wattr[JOB_ATR_exec_mics], NULL, NULL, - mic_str, + mic_str.c_str(), 0); - - free(mic_str); } if (gpu_list.size() != 0) { - if ((rc = translate_howl_to_string(gpu_list, EMsg, &NCount, &gpu_str, NULL, FALSE)) != PBSE_NONE) + if ((rc = translate_howl_to_string(gpu_list, EMsg, NCount, gpu_str, NULL, false)) != PBSE_NONE) { free_nodes(pjob, spec); delete [] ard_array; @@ -5134,11 +5087,9 @@ int set_nodes( &pjob->ji_wattr[JOB_ATR_exec_gpus], NULL, NULL, - gpu_str, + gpu_str.c_str(), 0); /* O */ - free(gpu_str); - if (gpu_mode_rqstd != -1) gpu_flags = gpu_mode_rqstd; if (gpu_err_reset) @@ -5166,7 +5117,7 @@ int set_nodes( snprintf(log_buf, sizeof(log_buf), "job %s allocated %d nodes (nodelist=%.4000s)", pjob->ji_qs.ji_jobid, NCount, - *rtnlist); + node_list.c_str()); log_record(PBSEVENT_SCHED, PBS_EVENTCLASS_REQUEST, __func__, log_buf); } diff --git a/src/server/pbsd_main.c b/src/server/pbsd_main.c index 3fadc565ad..9ca5590d2a 100644 --- a/src/server/pbsd_main.c +++ b/src/server/pbsd_main.c @@ -128,7 +128,7 @@ #include "dis_init.h" #include "batch_request.h" #include "pbs_proto.h" -#include "u_tree.h" +#include "authorized_hosts.hpp" #include "utils.h" #include "threadpool.h" #include "../lib/Libutils/u_lock_ctl.h" /* lock_init */ @@ -321,6 +321,7 @@ int array_259_upgrade = FALSE; sem_t *job_clone_semaphore; /* used to track the number of job_clone_wt requests are outstanding */ acl_special limited_acls; +authorized_hosts auth_hosts; char server_localhost[PBS_MAXHOSTNAME + 1]; size_t localhost_len = PBS_MAXHOSTNAME; diff --git a/src/server/receive_mom_communication.c b/src/server/receive_mom_communication.c index ca86e834a0..9949edc4d0 100644 --- a/src/server/receive_mom_communication.c +++ b/src/server/receive_mom_communication.c @@ -93,6 +93,7 @@ #include "attribute.h" #include "mom_update.h" #include "u_tree.h" +#include "authorized_hosts.hpp" #include "net_cache.h" #include "threadpool.h" #include "../lib/Libnet/lib_net.h" @@ -379,12 +380,11 @@ void *svr_is_request( } ipaddr = args[1]; + + node = auth_hosts.get_authorized_node(ipaddr, mom_port); - if ((node = AVL_find(ipaddr, mom_port, ipaddrs)) != NULL) - { - node->lock_node(__func__, "AVL_find", LOGLEVEL); - } /* END if AVL_find != NULL) */ - else if (allow_any_mom) + if ((node == NULL) && + (allow_any_mom)) { const char *name = get_cached_nameinfo(&addr); diff --git a/src/server/req_runjob.c b/src/server/req_runjob.c index dfa352de9b..c2938b542a 100644 --- a/src/server/req_runjob.c +++ b/src/server/req_runjob.c @@ -904,6 +904,7 @@ int verify_moms_up( pjob->ji_rejectdest.push_back(nodestr); /* FAILURE - cannot lookup master compute host */ + free(hostlist); return(PBSE_RESCUNAV); } @@ -929,6 +930,7 @@ int verify_moms_up( pjob->ji_rejectdest.push_back(nodestr); /* FAILURE - cannot create socket for master compute host */ + free(hostlist); return(PBSE_RESCUNAV); } @@ -957,6 +959,7 @@ int verify_moms_up( pjob->ji_rejectdest.push_back(nodestr); /* FAILURE - cannot connect to master compute host */ + free(hostlist); return(PBSE_RESCUNAV); } @@ -1735,10 +1738,11 @@ job *chk_job_torun( /* check that execution host is actually set */ /* this can happen if running the job failed after stagein */ if (pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str == NULL) - { - req_reject(PBSE_EXECTHERE, 0, preq, NULL, "exec host not set but files staged in"); - return(NULL); - } + { + req_reject(PBSE_EXECTHERE, 0, preq, NULL, "exec host not set but files staged in"); + return(NULL); + } + /* specified destination must match exec_host */ if ((exec_host = strdup(pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str)) == NULL) { @@ -2091,9 +2095,9 @@ int assign_hosts( { unsigned int dummy; - char *list = NULL; - char *portlist = NULL; - char *hosttoalloc = NULL; + std::string list; + std::string portlist; + const char *hosttoalloc = NULL; resource *pres; int rc = 0; int procs=0; @@ -2120,8 +2124,8 @@ int assign_hosts( if ((given != NULL) && (given[0] != '\0')) { - hosttoalloc = get_correct_spec_string(given, pjob); - to_free = hosttoalloc; + to_free = get_correct_spec_string(given, pjob); + hosttoalloc = to_free; } else { @@ -2165,8 +2169,8 @@ int assign_hosts( (procs == 0) && (pjob->ji_wattr[JOB_ATR_req_information].at_val.at_ptr != NULL)) { - hosttoalloc = strdup(RESOURCE_20_FIND); - to_free = hosttoalloc; + to_free = strdup(RESOURCE_20_FIND); + hosttoalloc = to_free; } get_svr_attr_str(SRV_ATR_DefNode, &def_node); @@ -2197,19 +2201,18 @@ int assign_hosts( { /* fall back to 1 cluster node */ - hosttoalloc = strdup(PBS_DEFAULT_NODE); - to_free = hosttoalloc; + hosttoalloc = PBS_DEFAULT_NODE; } /* do we need to allocate the (cluster) node(s)? */ if (svr_totnodes != 0) { - rc = set_nodes(pjob, (char *)hosttoalloc, procs, &list, &portlist, FailHost, EMsg); + rc = set_nodes(pjob, (char *)hosttoalloc, procs, list, portlist, FailHost, EMsg); set_exec_host = 1; /* maybe new VPs, must set */ - hosttoalloc = list; + hosttoalloc = list.c_str(); } if (rc == 0) @@ -2236,7 +2239,7 @@ int assign_hosts( &pjob->ji_wattr[JOB_ATR_exec_port], NULL, NULL, - portlist, + portlist.c_str(), 0); /* O */ pjob->ji_modified = 1; @@ -2274,9 +2277,6 @@ int assign_hosts( { free_nodes(pjob); - if (list != NULL) - free(list); - sprintf(log_buf, "ALERT: job cannot allocate node '%s' (could not determine IP address for node)", pjob->ji_qs.ji_destin); @@ -2290,12 +2290,6 @@ int assign_hosts( } /* END if (rc == 0) */ - if (list != NULL) - free(list); - - if (portlist != NULL) - free(portlist); - if (to_free != NULL) free(to_free); diff --git a/src/server/svr_mail.c b/src/server/svr_mail.c index 1d178279bd..ea323675a3 100644 --- a/src/server/svr_mail.c +++ b/src/server/svr_mail.c @@ -440,6 +440,7 @@ void *send_the_mail( } /* END send_the_mail() */ + void set_output_files( job *pjob, diff --git a/src/test/Makefile.am b/src/test/Makefile.am index 7414919481..730fd5628c 100644 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -5,22 +5,22 @@ NUMA_DIRS = allocation machine numa_chip numa_core numa_socket numa_pci_device n endif SERVER_UT_DIRS = accounting array_func array_upgrade attr_recov batch_request completed_jobs_map \ - delete_all_tracker dis_read display_alps_status execution_slot_tracker \ - exiting_jobs geteusernam get_path_jobdata id_map incoming_request \ - issue_request job_attr_def job_container job_func job_qs_upgrade job_recov \ - job_recycler job_usage_info login_nodes mom_hierarchy_handler node_func \ - node_manager pbsd_init pbsd_main process_alps_status process_mom_update \ - process_request queue_func queue_recov queue_recycler receive_mom_communication \ - reply_send req_delete req_deletearray req_getcred req_gpuctrl req_holdarray \ - req_holdjob req_jobobit req_locate req_manager req_message req_modify \ - req_movejob req_quejob req_register req_rerun req_rescq req_runjob req_select \ - req_shutdown req_signal req_stat req_tokens req_track resc_def_all run_sched \ - stat_job svr_chk_owner svr_connect svr_format_job svr_func svr_jobfunc svr_mail \ - svr_movejob svr_recov svr_resccost svr_task user_info acl_special restricted_host mail_throttler \ - job_array job + delete_all_tracker dis_read display_alps_status execution_slot_tracker \ + exiting_jobs geteusernam get_path_jobdata id_map incoming_request \ + issue_request job_attr_def job_container job_func job_qs_upgrade job_recov \ + job_recycler job_usage_info login_nodes mom_hierarchy_handler node_func \ + node_manager pbsd_init pbsd_main process_alps_status process_mom_update \ + process_request queue_func queue_recov queue_recycler receive_mom_communication \ + reply_send req_delete req_deletearray req_getcred req_gpuctrl req_holdarray \ + req_holdjob req_jobobit req_locate req_manager req_message req_modify \ + req_movejob req_quejob req_register req_rerun req_rescq req_runjob req_select \ + req_shutdown req_signal req_stat req_tokens req_track resc_def_all run_sched \ + stat_job svr_chk_owner svr_connect svr_format_job svr_func svr_jobfunc svr_mail \ + svr_movejob svr_recov svr_resccost svr_task user_info acl_special \ + restricted_host mail_throttler job_array job LIBUTILS_UT_DIRS = u_MXML u_groups u_hash_map_structs u_lock_ctl u_misc u_mom_hierarchy u_mu \ - u_mutex_mgr u_putenv u_threadpool u_tree u_users u_xml + u_mutex_mgr u_putenv u_threadpool u_tree u_users u_xml authorized_hosts LIBATTR_UT_DIRS = attr_atomic attr_fn_acl attr_fn_arst attr_fn_b attr_fn_c attr_fn_freq \ attr_fn_hold attr_fn_intr attr_fn_intr attr_fn_l attr_fn_ll attr_fn_nppcu \ diff --git a/src/test/authorized_hosts/Makefile.am b/src/test/authorized_hosts/Makefile.am new file mode 100644 index 0000000000..fe381cb027 --- /dev/null +++ b/src/test/authorized_hosts/Makefile.am @@ -0,0 +1,4 @@ + +include ../Makefile_Utils.ut + +libuut_la_SOURCES = ${PROG_ROOT}/authorized_hosts.cpp diff --git a/src/test/authorized_hosts/scaffolding.c b/src/test/authorized_hosts/scaffolding.c new file mode 100644 index 0000000000..f0c69dac9d --- /dev/null +++ b/src/test/authorized_hosts/scaffolding.c @@ -0,0 +1,42 @@ +#include "license_pbs.h" /* See here for the software license */ +#include + +#include "pbs_nodes.h" + +pbsnode::pbsnode() {} +pbsnode::~pbsnode() {} + +const char *pbsnode::get_name() const + + { + return(this->nd_name.c_str()); + } + +void pbsnode::change_name( + + const char *name) + + { + this->nd_name = name; + } + +pbsnode *find_nodebyname(const char *node_name) + { + static pbsnode pnode; + + if ((!strcmp(node_name, "nalthis")) || + (!strcmp(node_name, "nalthis2")) || + (!strcmp(node_name, "roshar")) || + (!strcmp(node_name, "roshar2")) || + (!strcmp(node_name, "scadrial")) || + (!strcmp(node_name, "scadrial2")) || + (!strcmp(node_name, "elantris")) || + (!strcmp(node_name, "elantris2"))) + { + pnode.change_name(node_name); + return(&pnode); + } + + return(NULL); + } + diff --git a/src/test/authorized_hosts/test_uut.c b/src/test/authorized_hosts/test_uut.c new file mode 100644 index 0000000000..3f93bdeab3 --- /dev/null +++ b/src/test/authorized_hosts/test_uut.c @@ -0,0 +1,120 @@ +#include "license_pbs.h" /* See here for the software license */ +#include "lib_utils.h" +#include +#include + + +#include "pbs_error.h" +#include "pbs_nodes.h" +#include "authorized_hosts.hpp" + + + +START_TEST(test_basics) + { + authorized_hosts ah; + + ah.add_authorized_address(1, 1, "roshar"); + ah.add_authorized_address(1, 5, "roshar2"); + ah.add_authorized_address(2, 2, "scadrial"); + ah.add_authorized_address(2, 6, "scadrial2"); + ah.add_authorized_address(3, 3, "nalthis"); + ah.add_authorized_address(3, 7, "nalthis2"); + ah.add_authorized_address(4, 4, "elantris"); + ah.add_authorized_address(4, 8, "elantris2"); + + std::string list; + ah.list_authorized_hosts(list); + fail_unless(list.find(":1") != std::string::npos, list.c_str()); + fail_unless(list.find(":2") != std::string::npos, list.c_str()); + fail_unless(list.find(":3") != std::string::npos, list.c_str()); + fail_unless(list.find(":4") != std::string::npos, list.c_str()); + fail_unless(list.find(":5") != std::string::npos, list.c_str()); + fail_unless(list.find(":6") != std::string::npos, list.c_str()); + fail_unless(list.find(":7") != std::string::npos, list.c_str()); + fail_unless(list.find(":8") != std::string::npos, list.c_str()); + + for (int i = 1; i < 5; i++) + { + // Make sure we find what we just added + fail_unless(ah.is_authorized(i)); + fail_unless(ah.is_authorized(i, i)); + fail_unless(ah.get_authorized_node(i) != NULL); + fail_unless(ah.get_authorized_node(i, i) != NULL); + fail_unless(ah.get_authorized_node(i, i + 4) != NULL); + + // Make sure the port matching functions only work if the port matches + fail_unless(ah.get_authorized_node(i, i + 1) == NULL, "Found authorized node for %d:%d", i, i+1); + fail_unless(ah.is_authorized(i, i + 1) == false); + } + + fail_unless(!strcmp(ah.get_authorized_node(1, 1)->get_name(), "roshar")); + fail_unless(!strcmp(ah.get_authorized_node(1, 5)->get_name(), "roshar2")); + fail_unless(!strcmp(ah.get_authorized_node(2, 2)->get_name(), "scadrial")); + fail_unless(!strcmp(ah.get_authorized_node(2, 6)->get_name(), "scadrial2")); + fail_unless(!strcmp(ah.get_authorized_node(3, 3)->get_name(), "nalthis")); + fail_unless(!strcmp(ah.get_authorized_node(3, 7)->get_name(), "nalthis2")); + fail_unless(!strcmp(ah.get_authorized_node(4, 4)->get_name(), "elantris")); + fail_unless(!strcmp(ah.get_authorized_node(4, 8)->get_name(), "elantris2")); + + fail_unless(ah.remove_address(1, 1) == true); + fail_unless(ah.is_authorized(1) == true); // the other port is still there + fail_unless(ah.is_authorized(1, 5) == true); // the other port is still there + fail_unless(ah.is_authorized(1, 1) == false); + fail_unless(ah.remove_address(1, 5) == true); + fail_unless(ah.remove_address(1, 1) == false); + fail_unless(ah.remove_address(1, 5) == false); + fail_unless(ah.is_authorized(1) == false); + fail_unless(ah.is_authorized(1, 1) == false); + fail_unless(ah.is_authorized(1, 5) == false); + + // Random stuff shouldn't work + for (int i = 5; i < 9; i++) + { + fail_unless(ah.is_authorized(i) == false); + fail_unless(ah.is_authorized(i, i) == false); + fail_unless(ah.get_authorized_node(i) == NULL); + fail_unless(ah.get_authorized_node(i, i) == NULL); + } + + // Clear so everything fails + ah.clear(); + + for (int i = 1; i < 5; i++) + { + fail_unless(ah.is_authorized(i) == false); + fail_unless(ah.is_authorized(i, i) == false); + fail_unless(ah.get_authorized_node(i) == NULL); + fail_unless(ah.get_authorized_node(i, i) == NULL); + } + } +END_TEST + + + +Suite *authorized_hosts_suite(void) + { + Suite *s = suite_create("authorized_hosts_suite methods"); + TCase *tc_core = tcase_create("test_basics"); + tcase_add_test(tc_core, test_basics); + suite_add_tcase(s, tc_core); + + return s; + } + +void rundebug() + { + } + +int main(void) + { + int number_failed = 0; + SRunner *sr = NULL; + rundebug(); + sr = srunner_create(authorized_hosts_suite()); + srunner_set_log(sr, "authorized_hosts_suite.log"); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return number_failed; + } diff --git a/src/test/mom_comm/scaffolding.c b/src/test/mom_comm/scaffolding.c index 2ee79b0ac0..09cf622d7d 100644 --- a/src/test/mom_comm/scaffolding.c +++ b/src/test/mom_comm/scaffolding.c @@ -20,6 +20,7 @@ #include "complete_req.hpp" #include "pmix_operation.hpp" #include "pmix_tracker.hpp" +#include "authorized_hosts.hpp" #ifdef NVIDIA_GPUS @@ -665,3 +666,13 @@ bool req::is_per_task() const #endif +bool authorized_hosts::is_authorized(unsigned long addr) + { + return(true); + } + +void authorized_hosts::list_authorized_hosts(std::string &output) {} + +authorized_hosts::authorized_hosts() {} +authorized_hosts auth_hosts; + diff --git a/src/test/mom_main/scaffolding.c b/src/test/mom_main/scaffolding.c index 0f3d944313..46b6c37542 100644 --- a/src/test/mom_main/scaffolding.c +++ b/src/test/mom_main/scaffolding.c @@ -27,6 +27,7 @@ #include #include #include +#include "authorized_hosts.hpp" extern mom_hierarchy_t *mh; @@ -1124,3 +1125,14 @@ int mkdir_wrapper(const char *path, mode_t mode) return(0); } +bool authorized_hosts::is_authorized(unsigned long addr) + { + return(true); + } + +void authorized_hosts::list_authorized_hosts(std::string &output) {} + +void authorized_hosts::add_authorized_address(unsigned long addr, unsigned short port, const std::string &hostname) {} + +authorized_hosts::authorized_hosts() {} +authorized_hosts auth_hosts; diff --git a/src/test/mom_process_request/scaffolding.c b/src/test/mom_process_request/scaffolding.c index 899c25be78..158cc85a79 100644 --- a/src/test/mom_process_request/scaffolding.c +++ b/src/test/mom_process_request/scaffolding.c @@ -11,6 +11,7 @@ #include "log.h" /* LOG_BUF_SIZE */ #include "batch_request.h" /* batch_request */ #include "pbs_job.h" /* job */ +#include "authorized_hosts.hpp" time_t time_now; const char *msg_err_malloc = "malloc failed"; @@ -279,3 +280,12 @@ char * netaddr_long(long ap, char *out) } void req_cleanup_job(batch_request *preq) {} + +bool authorized_hosts::is_authorized(unsigned long addr) + { + return(true); + } + +authorized_hosts::authorized_hosts() {} +authorized_hosts auth_hosts; + diff --git a/src/test/mom_server/scaffolding.c b/src/test/mom_server/scaffolding.c index efe4b03997..17a306cf64 100644 --- a/src/test/mom_server/scaffolding.c +++ b/src/test/mom_server/scaffolding.c @@ -22,6 +22,7 @@ #include "machine.hpp" #include "dis.h" #include "mom_func.h" +#include "authorized_hosts.hpp" #define MAXLINE 1024 @@ -453,3 +454,16 @@ void report_node_varattrs(std::map &varattrs) {} void report_node_features(std::set &features) {} #endif +bool authorized_hosts::is_authorized(unsigned long addr) + { + return(true); + } + +void authorized_hosts::list_authorized_hosts(std::string &output) {} + +void authorized_hosts::add_authorized_address(unsigned long addr, unsigned short port, const std::string &hostname) {} + +void authorized_hosts::clear() {} + +authorized_hosts::authorized_hosts() {} +authorized_hosts auth_hosts; diff --git a/src/test/node_func/scaffolding.c b/src/test/node_func/scaffolding.c index 3922bbc336..e1cf4600f2 100644 --- a/src/test/node_func/scaffolding.c +++ b/src/test/node_func/scaffolding.c @@ -22,7 +22,7 @@ #include "threadpool.h" #include "mom_hierarchy_handler.h" #include "machine.hpp" - +#include "authorized_hosts.hpp" std::string attrname; std::string attrval; @@ -588,3 +588,13 @@ int node_status_list( { return(0); } + +void authorized_hosts::add_authorized_address(unsigned long addr, unsigned short port, const std::string &hostname) {} + +bool authorized_hosts::remove_address(unsigned long addr, unsigned short port) + { + return(true); + } + +authorized_hosts::authorized_hosts() {} +authorized_hosts auth_hosts; diff --git a/src/test/node_manager/scaffolding.c b/src/test/node_manager/scaffolding.c index b270ebba81..f7d3675ddd 100644 --- a/src/test/node_manager/scaffolding.c +++ b/src/test/node_manager/scaffolding.c @@ -22,6 +22,7 @@ #include "machine.hpp" #include "complete_req.hpp" #include "json/json.h" +#include "authorized_hosts.hpp" bool cray_enabled; @@ -1078,3 +1079,11 @@ bool have_incompatible_dash_l_resource(pbs_attribute *pattr) } int pbsnode::get_version() const {return 0;} + +pbsnode *authorized_hosts::get_authorized_node(unsigned long addr, unsigned short port) + { + return(NULL); + } + +authorized_hosts::authorized_hosts() {} +authorized_hosts auth_hosts; diff --git a/src/test/parse_config/scaffolding.c b/src/test/parse_config/scaffolding.c index 203e0121fb..375dffffe7 100644 --- a/src/test/parse_config/scaffolding.c +++ b/src/test/parse_config/scaffolding.c @@ -8,6 +8,7 @@ #include "u_tree.h" #include "log.h" #include "json/json.h" +#include "authorized_hosts.hpp" #define LOG_BUF_SIZE 16384 #define MAXLINE 1024 @@ -219,3 +220,9 @@ char *conf_res(char *resline, struct rm_attribute *attr) return(resline); } + +void authorized_hosts::add_authorized_address(unsigned long addr, unsigned short port, const std::string &hostname) {} + +authorized_hosts::authorized_hosts() {} +authorized_hosts auth_hosts; + diff --git a/src/test/pbsd_main/scaffolding.c b/src/test/pbsd_main/scaffolding.c index 51153dfc51..90110e8fe8 100644 --- a/src/test/pbsd_main/scaffolding.c +++ b/src/test/pbsd_main/scaffolding.c @@ -18,6 +18,7 @@ #include "mom_hierarchy_handler.h" #include "completed_jobs_map.h" #include "acl_special.hpp" +#include "authorized_hosts.hpp" bool exit_called = false; pthread_mutex_t *job_log_mutex; @@ -516,3 +517,5 @@ completed_jobs_map_class::~completed_jobs_map_class() {} void *remove_completed_jobs(void *vp) {return(NULL);} acl_special::acl_special() {} + +authorized_hosts::authorized_hosts() {} diff --git a/src/test/receive_mom_communication/scaffolding.c b/src/test/receive_mom_communication/scaffolding.c index 3900e4da18..0dcd691b81 100644 --- a/src/test/receive_mom_communication/scaffolding.c +++ b/src/test/receive_mom_communication/scaffolding.c @@ -16,6 +16,7 @@ #include "id_map.hpp" #include "node_manager.h" #include "pbs_ifl.h" +#include "authorized_hosts.hpp" id_map job_mapper; @@ -925,3 +926,11 @@ int Machine::initializeNVIDIADevices(hwloc_obj_t, hwloc_topology_t) {return(0);} #endif job::job() {} + +pbsnode *authorized_hosts::get_authorized_node(unsigned long addr, unsigned short port) + { + return(NULL); + } + +authorized_hosts::authorized_hosts() {} +authorized_hosts auth_hosts; diff --git a/src/test/req_runjob/scaffolding.c b/src/test/req_runjob/scaffolding.c index 50e67b6625..c86e06be2b 100644 --- a/src/test/req_runjob/scaffolding.c +++ b/src/test/req_runjob/scaffolding.c @@ -204,7 +204,7 @@ void release_req(struct work_task *pwt) exit(1); } - int set_nodes(job *pjob, char *spec, int procs, char **rtnlist, char **rtnportlist, char *FailHost, char *EMsg) + int set_nodes(job *pjob, const char *spec, int procs, std::string &node_list, std::string &portlist, char *FailHost, char *EMsg) { fprintf(stderr, "The call to set_nodes to be mocked!!\n"); exit(1); From 0df0f51bf3c2853f6862ce762be634f791899631 Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Thu, 13 Oct 2016 12:43:54 -0600 Subject: [PATCH 017/312] Syncing up release/promotion_level resolution --- buildutils/torque-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index 2aae11cf3e..f79a0a876a 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -245,7 +245,7 @@ then make check | ./parse_cov_results.py fi -if [[ ${version} =~ ^[0-9]+\.[0-9]+(\.?(a|b|r?c)[0-9]+|(\.[0-9]+)+(\.h[0-9]+)?)$ ]] +if [[ ${version} =~ ^[0-9]+\.[0-9]+((\.[0-9]+)*\.?(a|b|r?c)[0-9]+|(\.[0-9]+)+(\.h[0-9]+)?)$ ]] then package_version="${version}" release=1 From adfb686bbaa35c56c14015d974f253c008f26c56 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 13 Oct 2016 13:44:25 -0600 Subject: [PATCH 018/312] Fix a header being included in the wrong place. --- src/include/authorized_hosts.hpp | 1 - src/test/authorized_hosts/test_uut.c | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/authorized_hosts.hpp b/src/include/authorized_hosts.hpp index 0e14dbb36a..e6bd8287f7 100644 --- a/src/include/authorized_hosts.hpp +++ b/src/include/authorized_hosts.hpp @@ -5,7 +5,6 @@ #include #include "pbs_nodes.h" -#include class authorized_host_info { diff --git a/src/test/authorized_hosts/test_uut.c b/src/test/authorized_hosts/test_uut.c index 3f93bdeab3..0ad64156b6 100644 --- a/src/test/authorized_hosts/test_uut.c +++ b/src/test/authorized_hosts/test_uut.c @@ -2,6 +2,7 @@ #include "lib_utils.h" #include #include +#include #include "pbs_error.h" From 003bdd3b6f82a3f9362b114a8b682955acd47a48 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 13 Oct 2016 14:16:53 -0600 Subject: [PATCH 019/312] TRQ-3809. Fix a mutex that is left locked when it shouldn't be. --- src/server/job_func.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/server/job_func.c b/src/server/job_func.c index 1930aa5cae..007cb3cc50 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -457,8 +457,8 @@ void handle_aborted_job( /* * job_abt - abort a job * - * The job removed from the system and a mail message is sent - * to the job owner. + * The job is deleted and a mail message is sent to the job owner. + * NOTE: The job is always unlocked when this function exits. */ /* NOTE: this routine is called under the following conditions: @@ -591,7 +591,9 @@ int job_abt( unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); } - pjob = svr_find_job(job_id, TRUE); + + if ((pjob = svr_find_job(job_id, TRUE)) != NULL) + pjob_mutex.mark_as_locked(); } } @@ -649,7 +651,9 @@ int job_abt( unlock_ai_mutex(pa, __func__,(char *) "1", LOGLEVEL); } - pjob = svr_find_job(job_id, TRUE); + + if ((pjob = svr_find_job(job_id, TRUE)) != NULL) + pjob_mutex.mark_as_locked(); } } From 01456f5ae15303148c3e08b134c80899e720a447 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 13 Oct 2016 15:02:24 -0600 Subject: [PATCH 020/312] TRQ-3803. Fix an invalid read around resources. --- src/server/job_route.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/server/job_route.c b/src/server/job_route.c index 11fa592cc3..7e7d4cecc4 100644 --- a/src/server/job_route.c +++ b/src/server/job_route.c @@ -731,6 +731,11 @@ int initialize_procct( pbs_errno = PBSE_INTERNAL; return(ROUTE_PERM_FAILURE); } + + // We have to refresh our pointers after adding a resource entry because resources are in a vector + // now and it may have been re-allocated. + pprocsp = find_resc_entry(pattr, pprocs_def); + pnodesp = find_resc_entry(pattr, pnodes_def); } /* Finally the moment of truth. We have the nodes and procs resources. Add them From e6507b988a1afaa6194f9db9ba248b9232fafb1b Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 13 Oct 2016 15:47:05 -0600 Subject: [PATCH 021/312] TRQ-3796 - fix makefile conditionals --- src/daemon_client/Makefile.am | 22 ++++++++++++++-------- src/resmom/Makefile.am | 23 ++++++++++++++--------- src/server/Makefile.am | 14 +++++++++----- 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/src/daemon_client/Makefile.am b/src/daemon_client/Makefile.am index a72edbe7ec..d0576e6632 100644 --- a/src/daemon_client/Makefile.am +++ b/src/daemon_client/Makefile.am @@ -12,15 +12,21 @@ trqauthd_SOURCES = trq_auth_daemon.c trq_main.c trqauthd_LDFLAGS = -L../lib/Libpbs/.libs -ltorque install-exec-hook: - systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1 && \ - echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service" && \ - $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || \ - (echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd" && \ - $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd) + if systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1; then \ + if [ -f $(top_srcdir)/contrib/systemd/trqauthd.service ]; then \ + echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service"; \ + $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service; \ + fi \ + elif [ -f $(top_srcdir)/contrib/init.d/trqauthd ]; then \ + echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd"; \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd; \ + fi -$(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d - test -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || \ - echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf && chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf && \ - echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig + if [ -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ + echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ + chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ + echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig; \ + fi uninstall-hook: -rm -f $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || \ diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index 632a1590a1..2e5fb81308 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -72,16 +72,21 @@ install-exec-hook: rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qnoded$(program_suffix)$(EXEEXT) ln -s $(program_prefix)pbs_mom$(program_suffix)$(EXEEXT) \ $(DESTDIR)$(sbindir)/$(program_prefix)qnoded$(program_suffix)$(EXEEXT) - systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1 && \ - echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service" && \ - $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service || \ - (echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom" && \ - $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom) + if systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1; then \ + if [ -f $(top_srcdir)/contrib/systemd/pbs_mom.service ]; then \ + echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service"; \ + $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service; \ + fi \ + elif [ -f $(top_srcdir)/contrib/init.d/pbs_mom ]; then \ + echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom"; \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom; \ + fi -$(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d - test -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || \ - echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf && \ - chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf && \ - echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig + if [ -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ + echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ + chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ + echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig; \ + fi uninstall-hook: rm -f $(DESTDIR)$(PBS_ENVIRON) diff --git a/src/server/Makefile.am b/src/server/Makefile.am index 93232ec8c8..f41cf04fa4 100644 --- a/src/server/Makefile.am +++ b/src/server/Makefile.am @@ -54,11 +54,15 @@ install-exec-hook: rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qserverd$(program_suffix)$(EXEEXT) ln -s $(program_prefix)pbs_server$(program_suffix)$(EXEEXT) \ $(DESTDIR)$(sbindir)/$(program_prefix)qserverd$(program_suffix)$(EXEEXT) - systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1 && \ - echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service" && \ - $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service || \ - (echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server" && \ - $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server) + if systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1; then \ + if [ -f $(top_srcdir)/contrib/systemd/pbs_server.service ]; then \ + echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service"; \ + $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service; \ + fi \ + elif [ -f $(top_srcdir)/contrib/init.d/pbs_server ]; then \ + echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server"; \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server; \ + fi uninstall-hook: rm -f $(DESTDIR)$(PBS_ENVIRON) $(DESTDIR)$(PBS_SERVER_HOME)/server_priv/nodes From eff962c987b9c5a3d469f184db10b40c2084fd1f Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 13 Oct 2016 20:57:30 -0600 Subject: [PATCH 022/312] TRQ-3796 - provide makefiles with hints about who called them --- Makefile.am | 2 +- buildutils/torque-build | 3 +++ src/cmds/Makefile.am | 23 +++++++++++------- src/daemon_client/Makefile.am | 44 +++++++++++++++++++++-------------- src/resmom/Makefile.am | 44 +++++++++++++++++++++-------------- src/server/Makefile.am | 24 +++++++++++-------- 6 files changed, 85 insertions(+), 55 deletions(-) diff --git a/Makefile.am b/Makefile.am index ef94b45397..524c827d65 100644 --- a/Makefile.am +++ b/Makefile.am @@ -162,7 +162,7 @@ packages: tarball=$(PKGROOT)/$(PACKAGE_NAME)-$$pname-$$d-$$phost.tar.gz; \ pkgfile=./$(PACKAGE_NAME)-$$pname-$$d-$$phost.sh; \ echo "Building $$pkgfile ..."; \ - $(MAKE) DESTDIR=$$proot install_$$d >/dev/null || exit 1; \ + $(MAKE) FROM_MAKE_PACKAGES=1 DESTDIR=$$proot install_$$d >/dev/null || exit 1; \ case $$d in \ devel) :;; \ *) rm -f $$proot/$(libdir)/*a $$proot/$(libdir)/*.so;; \ diff --git a/buildutils/torque-build b/buildutils/torque-build index f79a0a876a..0091042c4a 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -216,6 +216,9 @@ then configure_options="$configure_options --enable-cgroups --with-hwloc-path=/usr/local/bin/" fi +# set so child makefiles know this +export FROM_TORQUE_BUILD=1 + echo $(git rev-parse HEAD) > current_hash sh autogen.sh ./configure $configure_options diff --git a/src/cmds/Makefile.am b/src/cmds/Makefile.am index ab4182888b..02631ddfcc 100644 --- a/src/cmds/Makefile.am +++ b/src/cmds/Makefile.am @@ -77,16 +77,23 @@ install-exec-hook: rm -f $(DESTDIR)$(bindir)/$(program_prefix)qnodes$(program_suffix)$(EXEEXT) ln -s $(program_prefix)pbsnodes$(program_suffix)$(EXEEXT) \ $(DESTDIR)$(bindir)/$(program_prefix)qnodes$(program_suffix)$(EXEEXT) - -$(install_sh) -d -m 755 $(DESTDIR)/etc/profile.d - test -f $(DESTDIR)/etc/profile.d/torque.sh || \ - echo "PATH=$(bindir):$(sbindir):\$$PATH" > $(DESTDIR)/etc/profile.d/torque.sh - test -f $(DESTDIR)/etc/profile.d/torque.csh || \ - echo "set path = ($(bindir) $(sbindir) \$$path)" > $(DESTDIR)/etc/profile.d/torque.csh - chmod 644 $(DESTDIR)/etc/profile.d/torque.sh $(DESTDIR)/etc/profile.d/torque.csh + if [ $(FROM_TORQUE_BUILD)0 -eq 0 ]; then \ + $(install_sh) -d -m 755 $(DESTDIR)/etc/profile.d || :; \ + if [ ! -f $(DESTDIR)/etc/profile.d/torque.sh ]; then \ + echo "PATH=$(bindir):$(sbindir):\$$PATH" > $(DESTDIR)/etc/profile.d/torque.sh; \ + chmod 644 $(DESTDIR)/etc/profile.d/torque.sh; \ + fi; \ + if [ ! -f $(DESTDIR)/etc/profile.d/torque.csh ]; then \ + echo "set path = ($(bindir) $(sbindir) \$$path)" > $(DESTDIR)/etc/profile.d/torque.csh; \ + chmod 644 $(DESTDIR)/etc/profile.d/torque.csh; \ + fi; \ + fi uninstall-hook: rm -f $(DESTDIR)$(PBS_DEFAULT_FILE) rm -f $(DESTDIR)$(bindir)/$(program_prefix)qnodes$(program_suffix)$(EXEEXT) - -rm -f $(DESTDIR)/etc/profile.d/torque.sh - -rm -f $(DESTDIR)/etc/profile.d/torque.csh + if [ $(FROM_TORQUE_BUILD)0 -eq 0 ]; then \ + rm -f $(DESTDIR)/etc/profile.d/torque.sh || :; \ + rm -f $(DESTDIR)/etc/profile.d/torque.csh || :; \ + fi diff --git a/src/daemon_client/Makefile.am b/src/daemon_client/Makefile.am index d0576e6632..3b6e238497 100644 --- a/src/daemon_client/Makefile.am +++ b/src/daemon_client/Makefile.am @@ -12,25 +12,33 @@ trqauthd_SOURCES = trq_auth_daemon.c trq_main.c trqauthd_LDFLAGS = -L../lib/Libpbs/.libs -ltorque install-exec-hook: - if systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1; then \ - if [ -f $(top_srcdir)/contrib/systemd/trqauthd.service ]; then \ - echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service"; \ - $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service; \ - fi \ - elif [ -f $(top_srcdir)/contrib/init.d/trqauthd ]; then \ - echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd"; \ - $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd; \ - fi - -$(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d - if [ -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ - echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig; \ + if [ $(FROM_TORQUE_BUILD)0 -eq 0 ]; then \ + if systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1; then \ + if [ -f $(top_srcdir)/contrib/systemd/trqauthd.service ]; then \ + echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service"; \ + $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service; \ + fi \ + elif [ -f $(top_srcdir)/contrib/init.d/trqauthd ]; then \ + echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd"; \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd; \ + fi; \ + $(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d || :; \ + if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ + echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ + chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ + if [ $(FROM_MAKE_PACKAGES)0 -eq 0 ]; then \ + /sbin/ldconfig; \ + fi; \ + fi; \ fi uninstall-hook: - -rm -f $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || \ - rm -f $(DESTDIR)/etc/init.d/trqauthd - -rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf - echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig + if [ $(FROM_TORQUE_BUILD)0 -eq 0 ]; then \ + rm -f $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || :; \ + rm -f $(DESTDIR)/etc/init.d/trqauthd || :; \ + rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ + if $(FROM_MAKE_PACKAGES)0 -eq 0; then \ + /sbin/ldconfig; \ + fi; \ + fi diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index 2e5fb81308..fb10fa5150 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -72,27 +72,35 @@ install-exec-hook: rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qnoded$(program_suffix)$(EXEEXT) ln -s $(program_prefix)pbs_mom$(program_suffix)$(EXEEXT) \ $(DESTDIR)$(sbindir)/$(program_prefix)qnoded$(program_suffix)$(EXEEXT) - if systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1; then \ - if [ -f $(top_srcdir)/contrib/systemd/pbs_mom.service ]; then \ - echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service"; \ - $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service; \ - fi \ - elif [ -f $(top_srcdir)/contrib/init.d/pbs_mom ]; then \ - echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom"; \ - $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom; \ - fi - -$(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d - if [ -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ - echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig; \ + if [ $(FROM_TORQUE_BUILD)0 -eq 0 ]; then \ + if systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1; then \ + if [ -f $(top_srcdir)/contrib/systemd/pbs_mom.service ]; then \ + echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service"; \ + $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service; \ + fi \ + elif [ -f $(top_srcdir)/contrib/init.d/pbs_mom ]; then \ + echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom"; \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom; \ + fi; \ + $(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d || :; \ + if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ + echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ + chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ + if [ $(FROM_MAKE_PACKAGES)0 -eq 0 ]; then \ + /sbin/ldconfig; \ + fi; \ + fi; \ fi uninstall-hook: rm -f $(DESTDIR)$(PBS_ENVIRON) rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qnoded$(program_suffix)$(EXEEXT) - -rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service || \ - rm -f $(DESTDIR)/etc/init.d/pbs_mom - -rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf - echo $(DESTDIR) | egrep -q tpackages\|rpmbuild || /sbin/ldconfig + if [ $(FROM_TORQUE_BUILD)0 -eq 0 ]; then \ + rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service || :; \ + rm -f $(DESTDIR)/etc/init.d/pbs_mom || :; \ + rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ + if $(FROM_MAKE_PACKAGES)0 -eq 0; then \ + /sbin/ldconfig; \ + fi; \ + fi diff --git a/src/server/Makefile.am b/src/server/Makefile.am index f41cf04fa4..334b352c04 100644 --- a/src/server/Makefile.am +++ b/src/server/Makefile.am @@ -54,19 +54,23 @@ install-exec-hook: rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qserverd$(program_suffix)$(EXEEXT) ln -s $(program_prefix)pbs_server$(program_suffix)$(EXEEXT) \ $(DESTDIR)$(sbindir)/$(program_prefix)qserverd$(program_suffix)$(EXEEXT) - if systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1; then \ - if [ -f $(top_srcdir)/contrib/systemd/pbs_server.service ]; then \ - echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service"; \ - $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service; \ - fi \ - elif [ -f $(top_srcdir)/contrib/init.d/pbs_server ]; then \ - echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server"; \ - $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server; \ + if [ $(FROM_TORQUE_BUILD)0 -eq 0 ]; then \ + if systemctl 2>/dev/null | grep -- -\.mount >/dev/null 2>&1; then \ + if [ -f $(top_srcdir)/contrib/systemd/pbs_server.service ]; then \ + echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service"; \ + $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service; \ + fi \ + elif [ -f $(top_srcdir)/contrib/init.d/pbs_server ]; then \ + echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server"; \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server; \ + fi; \ fi uninstall-hook: rm -f $(DESTDIR)$(PBS_ENVIRON) $(DESTDIR)$(PBS_SERVER_HOME)/server_priv/nodes rm -f $(DESTDIR)$(sbindir)/$(program_prefix)qserverd$(program_suffix)$(EXEEXT) - -rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_server.service || \ - rm -f $(DESTDIR)/etc/init.d/pbs_server + if [ $(FROM_TORQUE_BUILD)0 -eq 0 ]; then \ + rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_server.service || :; \ + rm -f $(DESTDIR)/etc/init.d/pbs_server || :; \ + fi From f0ba7cfd647284de3c54f68f088828a5016d7db2 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 14 Oct 2016 10:16:35 -0600 Subject: [PATCH 023/312] Fix a segfault. --- src/include/authorized_hosts.hpp | 18 ------------------ src/lib/Libutils/authorized_hosts.cpp | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/src/include/authorized_hosts.hpp b/src/include/authorized_hosts.hpp index e6bd8287f7..2acaa0f6db 100644 --- a/src/include/authorized_hosts.hpp +++ b/src/include/authorized_hosts.hpp @@ -6,24 +6,6 @@ #include "pbs_nodes.h" -class authorized_host_info - { - friend class authorized_hosts; - - std::string hostname; - std::set ports; - - public: - authorized_host_info() : hostname(), ports() {} - authorized_host_info(const std::string &host, unsigned short port) : hostname(host) - { - ports.insert(port); - } - - authorized_host_info( - const authorized_host_info &other) : hostname(other.hostname), ports(other.ports) {} - }; - class authorized_hosts { diff --git a/src/lib/Libutils/authorized_hosts.cpp b/src/lib/Libutils/authorized_hosts.cpp index 868b275c54..b81fc158be 100644 --- a/src/lib/Libutils/authorized_hosts.cpp +++ b/src/lib/Libutils/authorized_hosts.cpp @@ -244,16 +244,19 @@ bool authorized_hosts::remove_address( pthread_mutex_lock(&this->auth_mutex); std::map >::iterator it = this->auth_map.find(addr); - std::map::iterator map_it = it->second.find(port); - if (map_it != it->second.end()) + if (it != this->auth_map.end()) { - removed = true; - it->second.erase(map_it); - - // If there are no authorized ports left, remove this address completely - if (it->second.size() == 0) + std::map::iterator map_it = it->second.find(port); + if (map_it != it->second.end()) { - this->auth_map.erase(it); + removed = true; + it->second.erase(map_it); + + // If there are no authorized ports left, remove this address completely + if (it->second.size() == 0) + { + this->auth_map.erase(it); + } } } From eede5e16244a9ace05683ffd3610de1722720cb5 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 14 Oct 2016 10:49:48 -0600 Subject: [PATCH 024/312] Fix test condition in makefile. --- src/daemon_client/Makefile.am | 2 +- src/resmom/Makefile.am | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/daemon_client/Makefile.am b/src/daemon_client/Makefile.am index 3b6e238497..b516173e7a 100644 --- a/src/daemon_client/Makefile.am +++ b/src/daemon_client/Makefile.am @@ -37,7 +37,7 @@ uninstall-hook: rm -f $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || :; \ rm -f $(DESTDIR)/etc/init.d/trqauthd || :; \ rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ - if $(FROM_MAKE_PACKAGES)0 -eq 0; then \ + if [ $(FROM_MAKE_PACKAGES)0 -eq 0 ]; then \ /sbin/ldconfig; \ fi; \ fi diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index fb10fa5150..7c6db90e20 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -99,7 +99,7 @@ uninstall-hook: rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service || :; \ rm -f $(DESTDIR)/etc/init.d/pbs_mom || :; \ rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ - if $(FROM_MAKE_PACKAGES)0 -eq 0; then \ + if [ $(FROM_MAKE_PACKAGES)0 -eq 0 ]; then \ /sbin/ldconfig; \ fi; \ fi From bb3892fd48085f9aa2ef1881048541da4a0bc094 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 14 Oct 2016 14:45:02 -0600 Subject: [PATCH 025/312] TRQ-3806 - enable cgroup support by default in Torque builder --- buildutils/torque-build | 16 ++++++++++++--- buildutils/torque.adaptive.spec.el6 | 24 +++++++++++++++++++++- buildutils/torque.adaptive.spec.el7 | 26 +++++++++++++++++++----- buildutils/torque.adaptive.spec.sles12 | 28 ++++++++++++++++++++++++-- 4 files changed, 83 insertions(+), 11 deletions(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index 0091042c4a..e6cf5f16f7 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -14,6 +14,8 @@ usage() { echo " directory)" echo " -d|--debug Enable debug mode" echo " -q|--quiet Enable quiet mode" + echo " -p|--hwloc-path Path to the hwloc package (required by cgroups)" + echo " (default is /usr/local)" echo " -v|--verbose Enable verbose mode" echo echo " -t|--tarball Make the TORQUE distribution tarball" @@ -27,6 +29,7 @@ usage() { echo " /opt/coverity/default/bin/cov-build)" echo " -C|--no-coverity Do not run coverity (default)" echo "" + echo " -g|--enable-cgroups Enable cgroups" echo " -u|--unit-tests Run unit-tests (default)" echo " -U|--no-unit-tests Do not run unit tests" echo "" @@ -45,7 +48,8 @@ coverity=0 unit_tests=1 parallel=3 tarball=1 -cgroups=0 +cgroups=1 +hwloc_path=/usr/local configure_options="" # To be set later @@ -64,6 +68,11 @@ do root_path="${1}" shift ;; + -p|--hwloc-path) + shift + hwloc_path="${1}" + shift + ;; -n|--name-suffix) shift name_suffix="${1}" @@ -213,7 +222,7 @@ echo "Started ${0}, version ${version}, release ${release}." \ if [ $cgroups -ne 0 ] then - configure_options="$configure_options --enable-cgroups --with-hwloc-path=/usr/local/bin/" + configure_options="$configure_options --enable-cgroups --with-hwloc-path=$hwloc_path" fi # set so child makefiles know this @@ -262,7 +271,7 @@ then make -j${parallel} clean make -j${parallel} distclean sh autogen.sh - ./configure + ./configure $configure_options make distcheck -j${parallel} distdir="${distdir}" fi @@ -329,6 +338,7 @@ then --define "_sourcedir ${root_path}/buildutils/rpmbuild/SOURCES" \ --define "_specdir ${root_path}/buildutils/rpmbuild/SPECS" \ --define "_tmppath ${root_path}/buildutils/rpmbuild/tmp" \ + --define "_hwlocpath $hwloc_path" \ --define "version ${version}" \ --define "release ${release}.${distro}" \ --define "source_file ${source_file}" \ diff --git a/buildutils/torque.adaptive.spec.el6 b/buildutils/torque.adaptive.spec.el6 index d6a155257c..131e2d09be 100644 --- a/buildutils/torque.adaptive.spec.el6 +++ b/buildutils/torque.adaptive.spec.el6 @@ -63,9 +63,13 @@ unset CONFFLAGS \ CONFFLAGS_FILE=`mktemp %{_tmppath}/confflags.XXXX` \ ./configure --help 2>&1 > ${CONFFLAGS_FILE} \ + if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + fi \ if grep -q -- '--oldincludedir' "${CONFFLAGS_FILE}" \ then \ - CONFFLAGS='--oldincludedir=%{_oldincludedir}' \ + CONFFLAGS="${CONFFLAGS} --oldincludedir=%{_oldincludedir}" \ fi \ if grep -q -- '--datarootdir' ${CONFFLAGS_FILE} \ then \ @@ -305,6 +309,7 @@ %bcond_with top ### Features enabled by default +%bcond_without cgroups %bcond_without scp %bcond_without spool %bcond_without syslog @@ -312,6 +317,7 @@ ### Autoconf macro expansions %define ac_with_blcr --%{?with_blcr:en}%{!?with_blcr:dis}able-blcr %define ac_with_cpuset --%{?with_cpuset:en}%{!?with_cpuset:dis}able-cpuset +%define ac_with_cgroups --%{?with_cgroups:en}%{!?with_cgroups:dis}able-cgroups %define ac_with_drmaa --%{?with_drmaa:en}%{!?with_drmaa:dis}able-drmaa %define ac_with_munge --%{?with_munge:en}%{!?with_munge:dis}able-munge-auth %define ac_with_numa --%{?with_numa:en}%{!?with_numa:dis}able-numa-support @@ -327,6 +333,7 @@ %define breq_munge %{?with_munge:munge-devel} %define breq_pam %{?with_pam:pam-devel} %define breq_scp %{?with_scp:/usr/bin/scp} +%define breq_cgroups %{?with_cgroups:libcgroup} %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} # Missing: # libcpuset -> libcpuset.so.* @@ -429,6 +436,9 @@ Suite. %package %{common_sub} Group: Applications/System Summary: TORQUE Common Files +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif %description %{common_sub} Common files shared by TORQUE Server, Client, and MOM packages @@ -445,6 +455,9 @@ Obsoletes: %{community_client_pkg} < %{version}-%{release} Provides: %{community_client_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_client_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif Provides: %{community_top_pkg}-docs = %{version}-%{release} Obsoletes: %{community_top_pkg}-docs < %{version}-%{release} Requires: %{common_pkg} @@ -463,6 +476,9 @@ Obsoletes: %{community_server_pkg} < %{version}-%{release} Provides: %{community_server_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_server_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif Requires: %{common_pkg} Requires: %{client_pkg} @@ -484,6 +500,9 @@ Obsoletes: %{community_devel_pkg} < %{version}-%{release} Provides: %{community_devel_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_devel_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif %description %{devel_sub} Development headers and libraries for TORQUE @@ -505,6 +524,9 @@ Obsoletes: %{community_mom_pkg} < %{version}-%{release} Provides: %{community_mom_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_mom_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif %description %{mom_sub} TORQUE MOM provides the agent necessary for each compute node in a diff --git a/buildutils/torque.adaptive.spec.el7 b/buildutils/torque.adaptive.spec.el7 index 242cd5b6b6..4e1bfb09c7 100644 --- a/buildutils/torque.adaptive.spec.el7 +++ b/buildutils/torque.adaptive.spec.el7 @@ -63,9 +63,13 @@ unset CONFFLAGS \ CONFFLAGS_FILE=`mktemp %{_tmppath}/confflags.XXXX` \ ./configure --help 2>&1 > ${CONFFLAGS_FILE} \ + if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + fi \ if grep -q -- '--oldincludedir' "${CONFFLAGS_FILE}" \ then \ - CONFFLAGS='--oldincludedir=%{_oldincludedir}' \ + CONFFLAGS="${CONFFLAGS} --oldincludedir=%{_oldincludedir}" \ fi \ if grep -q -- '--datarootdir' ${CONFFLAGS_FILE} \ then \ @@ -297,7 +301,6 @@ ### Features disabled by default %bcond_with blcr %bcond_with cpuset -%bcond_with cgroups %bcond_with libcpuset %bcond_with memacct %bcond_with munge @@ -306,6 +309,7 @@ %bcond_with top ### Features enabled by default +%bcond_without cgroups %bcond_without scp %bcond_without spool %bcond_without syslog @@ -313,7 +317,7 @@ ### Autoconf macro expansions %define ac_with_blcr --%{?with_blcr:en}%{!?with_blcr:dis}able-blcr %define ac_with_cpuset --%{?with_cpuset:en}%{!?with_cpuset:dis}able-cpuset -%define ac_with_cgroups --%{?with_cgroups:en}%{!?with_cgroups:dis}able-cgroups +%define ac_with_cgroups --%{?with_cgroups:en}%{!?with_cgroups:dis}able-cgroups %define ac_with_drmaa --%{?with_drmaa:en}%{!?with_drmaa:dis}able-drmaa %define ac_with_munge --%{?with_munge:en}%{!?with_munge:dis}able-munge-auth %define ac_with_numa --%{?with_numa:en}%{!?with_numa:dis}able-numa-support @@ -329,7 +333,7 @@ %define breq_munge %{?with_munge:munge-devel} %define breq_pam %{?with_pam:pam-devel} %define breq_scp %{?with_scp:/usr/bin/scp} -%define req_cgroups %{?with_cgroups:%{rootbin}/lssubsys} +%define breq_cgroups %{?with_cgroups:libcgroup} %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} # Missing: # libcpuset -> libcpuset.so.* @@ -433,7 +437,7 @@ Suite. Group: Applications/System Summary: TORQUE Common Files %if %{with cgroups} -Requires: %{req_cgroups} +Requires: %{breq_cgroups} %endif %description %{common_sub} @@ -451,6 +455,9 @@ Obsoletes: %{community_client_pkg} < %{version}-%{release} Provides: %{community_client_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_client_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif Provides: %{community_top_pkg}-docs = %{version}-%{release} Obsoletes: %{community_top_pkg}-docs < %{version}-%{release} Requires: %{common_pkg} @@ -469,6 +476,9 @@ Obsoletes: %{community_server_pkg} < %{version}-%{release} Provides: %{community_server_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_server_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif Requires: %{common_pkg} Requires: %{client_pkg} @@ -490,6 +500,9 @@ Obsoletes: %{community_devel_pkg} < %{version}-%{release} Provides: %{community_devel_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_devel_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif %description %{devel_sub} Development headers and libraries for TORQUE @@ -511,6 +524,9 @@ Obsoletes: %{community_mom_pkg} < %{version}-%{release} Provides: %{community_mom_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_mom_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif %description %{mom_sub} TORQUE MOM provides the agent necessary for each compute node in a diff --git a/buildutils/torque.adaptive.spec.sles12 b/buildutils/torque.adaptive.spec.sles12 index e9c0e311c7..4e1bfb09c7 100644 --- a/buildutils/torque.adaptive.spec.sles12 +++ b/buildutils/torque.adaptive.spec.sles12 @@ -63,9 +63,13 @@ unset CONFFLAGS \ CONFFLAGS_FILE=`mktemp %{_tmppath}/confflags.XXXX` \ ./configure --help 2>&1 > ${CONFFLAGS_FILE} \ + if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + fi \ if grep -q -- '--oldincludedir' "${CONFFLAGS_FILE}" \ then \ - CONFFLAGS='--oldincludedir=%{_oldincludedir}' \ + CONFFLAGS="${CONFFLAGS} --oldincludedir=%{_oldincludedir}" \ fi \ if grep -q -- '--datarootdir' ${CONFFLAGS_FILE} \ then \ @@ -305,6 +309,7 @@ %bcond_with top ### Features enabled by default +%bcond_without cgroups %bcond_without scp %bcond_without spool %bcond_without syslog @@ -312,6 +317,7 @@ ### Autoconf macro expansions %define ac_with_blcr --%{?with_blcr:en}%{!?with_blcr:dis}able-blcr %define ac_with_cpuset --%{?with_cpuset:en}%{!?with_cpuset:dis}able-cpuset +%define ac_with_cgroups --%{?with_cgroups:en}%{!?with_cgroups:dis}able-cgroups %define ac_with_drmaa --%{?with_drmaa:en}%{!?with_drmaa:dis}able-drmaa %define ac_with_munge --%{?with_munge:en}%{!?with_munge:dis}able-munge-auth %define ac_with_numa --%{?with_numa:en}%{!?with_numa:dis}able-numa-support @@ -327,6 +333,7 @@ %define breq_munge %{?with_munge:munge-devel} %define breq_pam %{?with_pam:pam-devel} %define breq_scp %{?with_scp:/usr/bin/scp} +%define breq_cgroups %{?with_cgroups:libcgroup} %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} # Missing: # libcpuset -> libcpuset.so.* @@ -429,6 +436,9 @@ Suite. %package %{common_sub} Group: Applications/System Summary: TORQUE Common Files +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif %description %{common_sub} Common files shared by TORQUE Server, Client, and MOM packages @@ -445,6 +455,9 @@ Obsoletes: %{community_client_pkg} < %{version}-%{release} Provides: %{community_client_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_client_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif Provides: %{community_top_pkg}-docs = %{version}-%{release} Obsoletes: %{community_top_pkg}-docs < %{version}-%{release} Requires: %{common_pkg} @@ -463,6 +476,9 @@ Obsoletes: %{community_server_pkg} < %{version}-%{release} Provides: %{community_server_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_server_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif Requires: %{common_pkg} Requires: %{client_pkg} @@ -484,6 +500,9 @@ Obsoletes: %{community_devel_pkg} < %{version}-%{release} Provides: %{community_devel_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_devel_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif %description %{devel_sub} Development headers and libraries for TORQUE @@ -505,6 +524,9 @@ Obsoletes: %{community_mom_pkg} < %{version}-%{release} Provides: %{community_mom_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_mom_pkg}(x86-64) < %{version}-%{release} %endif +%if %{with cgroups} +Requires: %{breq_cgroups} +%endif %description %{mom_sub} TORQUE MOM provides the agent necessary for each compute node in a @@ -529,7 +551,9 @@ CXXFLAGS="-g3 -O0" --without-tcl \ %{ac_with_scp} %{ac_with_syslog} \ %{ac_with_munge} %{ac_with_pam} \ - %{ac_with_blcr} %{ac_with_cpuset} %{ac_with_spool} %{?acflags} + %{ac_with_blcr} %{ac_with_cpuset} \ + %{ac_with_cgroups} %{ac_with_spool} %{?acflags} + %{__make} clean %{__make} %{?_smp_mflags} %{?mflags} From f7431642203142d262c22b9d1455786439cfc894 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 14 Oct 2016 17:04:32 -0600 Subject: [PATCH 026/312] Fix some more potential memory leaks. --- src/include/pbs_job.h | 15 ++++++--- src/server/job_func.c | 37 ++++++++++++--------- src/server/job_func.h | 4 --- src/server/node_manager.c | 2 +- src/server/req_jobobit.c | 4 +-- src/server/req_runjob.c | 16 ++++----- src/server/req_runjob.h | 2 +- src/server/req_stat.c | 4 +-- src/server/req_stat.h | 4 +-- src/server/svr_jobfunc.c | 69 ++++++++++++--------------------------- src/server/svr_mail.c | 27 ++++++++------- src/server/svr_movejob.c | 37 ++++++++++----------- 12 files changed, 100 insertions(+), 121 deletions(-) diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index e6a710bc80..985ac92416 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -1084,12 +1084,17 @@ task *task_find(job *pjob, tm_task_id taskid); #define LOCUTION_SIZE 20 -typedef struct send_job_request +class send_job_request { - char *jobid; - int move_type; - void *data; - } send_job_request; + public: + std::string jobid; + int move_type; + void *data; + + send_job_request() : jobid(), move_type(MOVE_TYPE_Move), data(NULL) + { + } + }; diff --git a/src/server/job_func.c b/src/server/job_func.c index 007cb3cc50..e0ce613c9c 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -161,7 +161,7 @@ extern int LOGLEVEL; extern completed_jobs_map_class completed_jobs_map; -int conn_qsub(char *, long, char *); +int conn_qsub(const char *, long, char *); /* External functions */ @@ -252,7 +252,7 @@ static void send_qsub_delmsg( static int remtree( - char *dirname) + const char *dirname) { DIR *dir; @@ -677,9 +677,9 @@ int job_abt( int conn_qsub( - char *hostname, /* I */ - long port, /* I */ - char *EMsg) /* O (optional,minsize=1024) */ + const char *hostname, /* I */ + long port, /* I */ + char *EMsg) /* O (optional,minsize=1024) */ { pbs_net_t hostaddr; @@ -2209,16 +2209,23 @@ struct pbs_queue *get_jobs_queue( +/* + * Checks if this hostname is in an exec_host-style list + * + * @param hostname - the hostname we're looking for + * @param externals - the exec_host-style list we're searching + * @return true if found, false otherwise + */ -/*static*/ int hostname_in_externals( +bool hostname_in_externals( - char *hostname, - char *externals) + const char *hostname, + const char *externals) { - char *ptr = strstr(externals, hostname); - int found = FALSE; - char *end_word; + const char *ptr = strstr(externals, hostname); + bool found = false; + const char *end_word; if (ptr != NULL) { @@ -2229,9 +2236,9 @@ struct pbs_queue *get_jobs_queue( found = FALSE; else if ((*end_word != '\0') && (*end_word != '+')) - found = FALSE; + found = false; else - found = TRUE; + found = true; } return(found); @@ -2275,7 +2282,7 @@ int fix_external_exec_hosts( *slash = '\0'; /* if we find a match, copy in that exec host entry */ - if (hostname_in_externals(exec_ptr, externals) == TRUE) + if (hostname_in_externals(exec_ptr, externals) == true) { /* capture the first external as my mother superior */ if (pjob->ji_qs.ji_destin[0] == '\0') @@ -2342,7 +2349,7 @@ int fix_cray_exec_hosts( *slash = '\0'; /* if the hostname isn't in the externals, copy it in */ - if (hostname_in_externals(exec_ptr, external) == FALSE) + if (hostname_in_externals(exec_ptr, external) == false) { if (slash != NULL) *slash = '/'; diff --git a/src/server/job_func.h b/src/server/job_func.h index 2e8b6f3acb..f96694c87d 100644 --- a/src/server/job_func.h +++ b/src/server/job_func.h @@ -8,8 +8,6 @@ struct batch_request; int job_abt(struct job **pjobp, const char *text, bool depedentjob=0); -int conn_qsub(char *hostname, long port, char *EMsg); - struct job *copy_job(struct job *parent); struct job *job_clone(struct job *template_job, struct job_array *pa, int taskid); @@ -30,8 +28,6 @@ struct job_array *get_jobs_array(struct job **pjob); struct pbs_queue *get_jobs_queue(struct job **pjob_ptr); -int hostname_in_externals(char *hostname, char *externals); - int fix_external_exec_hosts(struct job *pjob); int fix_cray_exec_hosts(struct job *pjob); diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 56b03c5557..b9928b3480 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -188,7 +188,7 @@ extern int SvrNodeCt; extern int multi_mom; -const int network_fail_wait_time = 300; +const int network_fail_wait_time = 300; #define SKIP_NONE 0 #define SKIP_EXCLUSIVE 1 diff --git a/src/server/req_jobobit.c b/src/server/req_jobobit.c index dbeb5026b7..cde783afb0 100644 --- a/src/server/req_jobobit.c +++ b/src/server/req_jobobit.c @@ -1896,7 +1896,7 @@ int handle_complete_first_time( if (LOGLEVEL >= 7) { sprintf(log_buf, "adding job to completed_jobs_map from %s", __func__); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, jid.c_str(), log_buf); } // add job id and clean up time for processing by cleanup task @@ -3714,7 +3714,7 @@ int req_jobobit( /* clear suspended flag if it was set */ pjob->ji_qs.ji_svrflags &= ~JOB_SVFLG_Suspend; - if ((rc = update_substate_from_exit_status(pjob, &alreadymailed,mailbuf)) != PBSE_NONE) + if ((rc = update_substate_from_exit_status(pjob, &alreadymailed, mailbuf)) != PBSE_NONE) return(rc); /* What do we now do with the job... */ diff --git a/src/server/req_runjob.c b/src/server/req_runjob.c index c2938b542a..b7d985f834 100644 --- a/src/server/req_runjob.c +++ b/src/server/req_runjob.c @@ -127,7 +127,7 @@ /* External Functions Called: */ -extern int send_job_work(char *job_id,char *,int,int *,struct batch_request *); +extern int send_job_work(char *job_id, const char *,int,int *,struct batch_request *); extern void set_resc_assigned(job *, enum batch_op); extern struct batch_request *cpy_stage(struct batch_request *, job *, enum job_atr, int); @@ -1422,10 +1422,10 @@ int svr_strtjob2( void finish_sendmom( - char *job_id, + const char *job_id, struct batch_request *preq, long start_time, - char *node_name, + const char *node_name, int status, int mom_err) @@ -1877,12 +1877,12 @@ job *chk_job_torun( int set_job_exec_info( - job *pjob, - char *list) + job *pjob, + const char *list) { char ms[PBS_MAXHOSTNAME]; - char *ptr; + const char *ptr; int i; struct pbsnode *pnode; struct in_addr hostaddr; @@ -2208,7 +2208,7 @@ int assign_hosts( if (svr_totnodes != 0) { - rc = set_nodes(pjob, (char *)hosttoalloc, procs, list, portlist, FailHost, EMsg); + rc = set_nodes(pjob, hosttoalloc, procs, list, portlist, FailHost, EMsg); set_exec_host = 1; /* maybe new VPs, must set */ @@ -2255,7 +2255,7 @@ int assign_hosts( (pjob->ji_wattr[JOB_ATR_login_node_id].at_val.at_str != NULL)) tmp = parse_servername(pjob->ji_wattr[JOB_ATR_login_node_id].at_val.at_str, &dummy); else - tmp = parse_servername((char *)hosttoalloc, &dummy); + tmp = parse_servername(hosttoalloc, &dummy); snprintf(pjob->ji_qs.ji_destin, sizeof(pjob->ji_qs.ji_destin), "%s", tmp); diff --git a/src/server/req_runjob.h b/src/server/req_runjob.h index 34cb5330df..25b11ad34b 100644 --- a/src/server/req_runjob.h +++ b/src/server/req_runjob.h @@ -23,7 +23,7 @@ int svr_startjob(job *pjob, struct batch_request *preq, char *FailHost, char *EM /* static int svr_strtjob2(job *pjob, struct batch_request *preq); */ -void finish_sendmom(char *job_id, struct batch_request *preq, long start_time, char *node_name, int status, int); +void finish_sendmom(const char *job_id, struct batch_request *preq, long start_time, const char *node_name, int status, int); /* static job *chk_job_torun(struct batch_request *preq, int setnn); */ diff --git a/src/server/req_stat.c b/src/server/req_stat.c index 32ab77f304..e397b1ff5a 100644 --- a/src/server/req_stat.c +++ b/src/server/req_stat.c @@ -737,7 +737,7 @@ void req_stat_job_step2( int stat_to_mom( - char *job_id, + const char *job_id, struct stat_cntl *cntl) /* M */ { @@ -974,7 +974,7 @@ void stat_update( void stat_mom_job( - char *job_id) + const char *job_id) { struct stat_cntl *cntl; diff --git a/src/server/req_stat.h b/src/server/req_stat.h index ebdb2f2d4d..585bc86766 100644 --- a/src/server/req_stat.h +++ b/src/server/req_stat.h @@ -11,9 +11,9 @@ int req_stat_job(struct batch_request *preq); -int stat_to_mom(char *job_id, struct stat_cntl *cntl); +int stat_to_mom(const char *job_id, struct stat_cntl *cntl); -void stat_mom_job(char *jobid); +void stat_mom_job(const char *jobid); void stat_update(struct batch_request *preq, struct stat_cntl *cntl); diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index 44a226d61f..52a9e95c94 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -3929,6 +3929,7 @@ static void correct_ct() } /* END correct_ct() */ + int lock_ji_mutex( job *pjob, @@ -3937,16 +3938,12 @@ int lock_ji_mutex( int logging) { - int rc = PBSE_NONE; - char *err_msg = NULL; - char stub_msg[] = "no pos"; + int rc = PBSE_NONE; + char err_msg[MSG_LEN_LONG]; if (logging >= 10) { - err_msg = (char *)calloc(1, MSG_LEN_LONG); - if (msg == NULL) - msg = stub_msg; - snprintf(err_msg, MSG_LEN_LONG, "locking %s in method %s-%s", pjob->ji_qs.ji_jobid, id, msg); + snprintf(err_msg, sizeof(err_msg), "locking %s in method %s-%s", pjob->ji_qs.ji_jobid, id, msg); log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); } @@ -3956,8 +3953,8 @@ int lock_ji_mutex( { if (logging >= 20) { - snprintf(err_msg, MSG_LEN_LONG, "ALERT: cannot lock job %s mutex in method %s", - pjob->ji_qs.ji_jobid, id); + snprintf(err_msg, sizeof(err_msg), "ALERT: cannot lock job %s mutex in method %s", + pjob->ji_qs.ji_jobid, id); log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); } rc = PBSE_MUTEX; @@ -3969,9 +3966,6 @@ int lock_ji_mutex( log_err(rc, __func__, "Uninitialized mutex pass to pthread_mutex_lock!"); } - if (err_msg != NULL) - free(err_msg); - return rc; } @@ -3985,16 +3979,12 @@ int unlock_ji_mutex( int logging) { - int rc = PBSE_NONE; - char *err_msg = NULL; - char stub_msg[] = "no pos"; + int rc = PBSE_NONE; + char err_msg[MSG_LEN_LONG]; if (logging >= 10) { - err_msg = (char *)calloc(1, MSG_LEN_LONG); - if (msg == NULL) - msg = stub_msg; - snprintf(err_msg, MSG_LEN_LONG, "unlocking %s in method %s-%s", pjob->ji_qs.ji_jobid, id, msg); + snprintf(err_msg, sizeof(err_msg), "unlocking %s in method %s-%s", pjob->ji_qs.ji_jobid, id, msg); log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); } @@ -4004,8 +3994,8 @@ int unlock_ji_mutex( { if (logging >= 20) { - snprintf(err_msg, MSG_LEN_LONG, "ALERT: cannot unlock job %s mutex in method %s", - pjob->ji_qs.ji_jobid, id); + snprintf(err_msg, sizeof(err_msg), "ALERT: cannot unlock job %s mutex in method %s", + pjob->ji_qs.ji_jobid, id); log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); } rc = PBSE_MUTEX; @@ -4017,9 +4007,6 @@ int unlock_ji_mutex( log_err(rc, __func__, "Uninitialized mutex pass to pthread_mutex_unlock!"); } - if (err_msg != NULL) - free(err_msg); - return rc; } @@ -4032,16 +4019,12 @@ int lock_ai_mutex( int logging) { - int rc = PBSE_NONE; - char *err_msg = NULL; - char stub_msg[] = "no pos"; + int rc = PBSE_NONE; + char err_msg[MSG_LEN_LONG]; if (logging >= 10) { - err_msg = (char *)calloc(1, MSG_LEN_LONG); - if (msg == NULL) - msg = stub_msg; - snprintf(err_msg, MSG_LEN_LONG, "locking %s in method %s-%s", pa->ai_qs.parent_id, id, msg); + snprintf(err_msg, sizeof(err_msg), "locking %s in method %s-%s", pa->ai_qs.parent_id, id, msg); log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); } @@ -4049,16 +4032,13 @@ int lock_ai_mutex( { if (logging >= 20) { - snprintf(err_msg, MSG_LEN_LONG, "ALERT: cannot lock job array %s mutex in method %s", - pa->ai_qs.parent_id, id); + snprintf(err_msg, sizeof(err_msg), "ALERT: cannot lock job array %s mutex in method %s", + pa->ai_qs.parent_id, id); log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); } rc = PBSE_MUTEX; } - if (err_msg != NULL) - free(err_msg); - return rc; } @@ -4071,16 +4051,12 @@ int unlock_ai_mutex( int logging) { - int rc = PBSE_NONE; - char *err_msg = NULL; - char stub_msg[] = "no pos"; + int rc = PBSE_NONE; + char err_msg[MSG_LEN_LONG]; if (logging >= 10) { - err_msg = (char *)calloc(1, MSG_LEN_LONG); - if (msg == NULL) - msg = stub_msg; - snprintf(err_msg, MSG_LEN_LONG, "unlocking %s in method %s-%s", pa->ai_qs.parent_id, id, msg); + snprintf(err_msg, sizeof(err_msg), "unlocking %s in method %s-%s", pa->ai_qs.parent_id, id, msg); log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); } @@ -4088,16 +4064,13 @@ int unlock_ai_mutex( { if (logging >= 20) { - snprintf(err_msg, MSG_LEN_LONG, "ALERT: cannot unlock job array %s mutex in method %s", - pa->ai_qs.parent_id, id); + snprintf(err_msg, sizeof(err_msg), "ALERT: cannot unlock job array %s mutex in method %s", + pa->ai_qs.parent_id, id); log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); } rc = PBSE_MUTEX; } - if (err_msg != NULL) - free(err_msg); - return rc; } diff --git a/src/server/svr_mail.c b/src/server/svr_mail.c index ea323675a3..7d297a7183 100644 --- a/src/server/svr_mail.c +++ b/src/server/svr_mail.c @@ -554,11 +554,12 @@ void send_email_batch( void svr_mailowner_with_message( - job *pjob, /* I */ - int mailpoint, /* note, single character */ - int force, /* if set to MAIL_FORCE, force mail delivery */ - const char *text, /* text to mail. */ + job *pjob, /* I */ + int mailpoint, /* note, single character */ + int force, /* if set to MAIL_FORCE, force mail delivery */ + const char *text, /* text to mail. */ const char *msg) /* Optional extra message */ + { if((text == NULL)||(*text == '\0')) { @@ -568,18 +569,16 @@ void svr_mailowner_with_message( { return svr_mailowner(pjob,mailpoint,force,text); } - char *newMsg = (char *)malloc(strlen(text) + strlen(msg) + 2); - if(newMsg == NULL) - { - return; - } - strcpy(newMsg,text); - strcat(newMsg,"\n"); - strcat(newMsg,msg); - svr_mailowner(pjob,mailpoint,force,newMsg); - free(newMsg); + + std::string newMsg(text); + newMsg += "\n"; + newMsg += msg; + + svr_mailowner(pjob, mailpoint, force, newMsg.c_str()); } + + void svr_mailowner( job *pjob, /* I */ diff --git a/src/server/svr_movejob.c b/src/server/svr_movejob.c index 1563777708..447d17e793 100644 --- a/src/server/svr_movejob.c +++ b/src/server/svr_movejob.c @@ -547,13 +547,13 @@ void finish_moving_processing( void finish_move_process( - char *job_id, - struct batch_request *preq, - long time, - char *node_name, - int status, - int type, - int mom_err) + char *job_id, + batch_request *preq, + long time, + const char *node_name, + int status, + int type, + int mom_err) { char log_buf[LOCAL_LOG_BUF_SIZE+1]; @@ -1011,8 +1011,8 @@ int commit_job_on_mom( int get_mom_node_version( - char *job_id, - int &version) + const char *job_id, + int &version) { job *pjob; @@ -1229,7 +1229,7 @@ int send_job_over_network_with_retries( int send_job_work( char *job_id, - char *node_name, /* I */ + const char *node_name, /* I */ int type, /* I */ int *my_err, /* O */ batch_request *preq) /* M */ @@ -1472,7 +1472,7 @@ void *send_job( { send_job_request *args = (send_job_request *)vp; - char *job_id = args->jobid; + char *job_id = strdup(args->jobid.c_str()); job *pjob; int type = args->move_type; /* move, route, or execute */ int local_errno = 0; @@ -1480,7 +1480,7 @@ void *send_job( char *node_name = NULL; - struct batch_request *preq = (struct batch_request *)args->data; + batch_request *preq = (struct batch_request *)args->data; pjob = svr_find_job(job_id, TRUE); @@ -1492,7 +1492,7 @@ void *send_job( { sprintf(log_buf,"about to send job - type=%d",type); - log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,job_id,log_buf); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, job_id, log_buf); } node_name = get_ms_name(*pjob); @@ -1502,11 +1502,12 @@ void *send_job( free(node_name); } - free(vp); - return(NULL); - } /* END send_job() */ + delete args; + free(job_id); + return(NULL); + } /* END send_job() */ @@ -1561,7 +1562,7 @@ int net_move( svr_setjobstate(jobp, JOB_STATE_TRANSIT, JOB_SUBSTATE_TRNOUT, TRUE); - args = (send_job_request *)calloc(1, sizeof(send_job_request)); + args = new send_job_request(); if (args != NULL) { @@ -1582,8 +1583,6 @@ int net_move( - - /* * should_retry_route - should the route be retried based on the error return * From 7b84ec715aa6c6a548ce8f0460bd4a591db5167c Mon Sep 17 00:00:00 2001 From: David Beer Date: Sat, 15 Oct 2016 09:06:07 -0600 Subject: [PATCH 027/312] Fix some scaffolding errors. --- src/test/job_func/test_uut.c | 3 ++- src/test/req_runjob/scaffolding.c | 6 ++---- src/test/svr_movejob/scaffolding.c | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/test/job_func/test_uut.c b/src/test/job_func/test_uut.c index f35b8a0e36..537d6067db 100644 --- a/src/test/job_func/test_uut.c +++ b/src/test/job_func/test_uut.c @@ -12,7 +12,7 @@ extern sem_t *job_clone_semaphore; extern attribute_def job_attr_def[]; -int hostname_in_externals(char *, char *); +int hostname_in_externals(const char *, const char *); int fix_cray_exec_hosts(job *); int fix_external_exec_hosts(job *); int change_external_job_name(job *); @@ -20,6 +20,7 @@ int split_job(job *); bool add_job_called = false; bool internal_job_id_exists(int internal_id); void job_free(job *pj, int use_recycle); +int conn_qsub(const char *hostname, long port, char *EMsg); //bool svr_job_purge_called = false; extern completed_jobs_map_class completed_jobs_map; diff --git a/src/test/req_runjob/scaffolding.c b/src/test/req_runjob/scaffolding.c index c86e06be2b..e8b056e00c 100644 --- a/src/test/req_runjob/scaffolding.c +++ b/src/test/req_runjob/scaffolding.c @@ -89,10 +89,8 @@ char *find_ts_node(void) exit(1); } -void stat_mom_job(char *job_id) +void stat_mom_job(const char *job_id) { - fprintf(stderr, "The call to stat_mom_job to be mocked!!\n"); - exit(1); } pbs_net_t get_hostaddr(int *local_errno, const char *hostname) @@ -245,7 +243,7 @@ void DIS_tcp_settimeout(long timeout) { } -int send_job_work(char *job_id, char *node_name, int type, int *my_err, struct batch_request *preq) +int send_job_work(char *job_id, const char *node_name, int type, int *my_err, struct batch_request *preq) { //returning failure return -1; diff --git a/src/test/svr_movejob/scaffolding.c b/src/test/svr_movejob/scaffolding.c index b08e902dea..23df732d23 100644 --- a/src/test/svr_movejob/scaffolding.c +++ b/src/test/svr_movejob/scaffolding.c @@ -97,7 +97,7 @@ void add_to_property_list(std::string &propstr, const char *token) -void finish_sendmom(char *job_id, batch_request *preq, long start_time, char *node_name, int status, int o) +void finish_sendmom(const char *job_id, batch_request *preq, long start_time, const char *node_name, int status, int o) { } From 944a9871f2b95d53bd7630aeb703f981356fce5e Mon Sep 17 00:00:00 2001 From: Jared Bristow Date: Sat, 15 Oct 2016 10:32:34 -0600 Subject: [PATCH 028/312] remove libcgroup-tools from el6 requirements --- buildutils/torque.adaptive.spec.el6 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque.adaptive.spec.el6 b/buildutils/torque.adaptive.spec.el6 index 131e2d09be..3bce650dab 100644 --- a/buildutils/torque.adaptive.spec.el6 +++ b/buildutils/torque.adaptive.spec.el6 @@ -333,7 +333,7 @@ %define breq_munge %{?with_munge:munge-devel} %define breq_pam %{?with_pam:pam-devel} %define breq_scp %{?with_scp:/usr/bin/scp} -%define breq_cgroups %{?with_cgroups:libcgroup} %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} +%define breq_cgroups %{?with_cgroups:libcgroup} %{?with_cgroups:hwloc >= 1.9.1} # Missing: # libcpuset -> libcpuset.so.* From 3e15d1504949fe5a9b9487034c6e3e444b459064 Mon Sep 17 00:00:00 2001 From: Jared Bristow Date: Fri, 14 Oct 2016 17:31:53 -0600 Subject: [PATCH 029/312] change spec file for sles12 to require libcgroup1 --- buildutils/torque.adaptive.spec.sles12 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque.adaptive.spec.sles12 b/buildutils/torque.adaptive.spec.sles12 index 4e1bfb09c7..176897df63 100644 --- a/buildutils/torque.adaptive.spec.sles12 +++ b/buildutils/torque.adaptive.spec.sles12 @@ -333,7 +333,7 @@ %define breq_munge %{?with_munge:munge-devel} %define breq_pam %{?with_pam:pam-devel} %define breq_scp %{?with_scp:/usr/bin/scp} -%define breq_cgroups %{?with_cgroups:libcgroup} %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} +%define breq_cgroups %{?with_cgroups:libcgroup1} %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} # Missing: # libcpuset -> libcpuset.so.* From afb931125a2578510f96b6c9b37ee64e89f43da1 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 17 Oct 2016 13:34:28 -0600 Subject: [PATCH 030/312] TRQ-3814. Fix a deadlock. --- src/server/node_manager.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index b9928b3480..52bd9a21e6 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -1385,7 +1385,6 @@ void stream_eof( } /* Before we mark this node down see if we can connect */ - np->lock_node(__func__, "parent", LOGLEVEL); conn = svr_connect(addr, port, &my_err, np, NULL); if (conn >= 0) { From 8f39e49852db7660976d80267663710d71829bd2 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 17 Oct 2016 15:23:58 -0600 Subject: [PATCH 031/312] TRQ-3796 - install proper init.d file under sles11 --- Makefile.am | 1 + src/daemon_client/Makefile.am | 8 ++++++-- src/resmom/Makefile.am | 8 ++++++-- src/server/Makefile.am | 10 +++++++--- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Makefile.am b/Makefile.am index 524c827d65..911e4e6318 100644 --- a/Makefile.am +++ b/Makefile.am @@ -28,6 +28,7 @@ EXTRA_DIST = acinclude.m4 \ torque.setup \ torque.spec \ buildutils/config.mk \ + buildutils/determine.distro \ buildutils/modulefiles.in \ buildutils/modulefiles.vers.in \ buildutils/pbs_mach_type \ diff --git a/src/daemon_client/Makefile.am b/src/daemon_client/Makefile.am index b516173e7a..57ca59bd18 100644 --- a/src/daemon_client/Makefile.am +++ b/src/daemon_client/Makefile.am @@ -19,8 +19,12 @@ install-exec-hook: $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/trqauthd.service $(DESTDIR)/usr/lib/systemd/system/trqauthd.service; \ fi \ elif [ -f $(top_srcdir)/contrib/init.d/trqauthd ]; then \ - echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd"; \ - $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/trqauthd $(DESTDIR)/etc/init.d/trqauthd; \ + prefix=''; \ + if [ "`$(top_srcdir)/buildutils/determine.distro`" = sles ]; then \ + prefix=suse. ; \ + fi; \ + echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/$${prefix}trqauthd $(DESTDIR)/etc/init.d/trqauthd"; \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/$${prefix}trqauthd $(DESTDIR)/etc/init.d/trqauthd; \ fi; \ $(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d || :; \ if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index 7c6db90e20..c6c9acb430 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -79,8 +79,12 @@ install-exec-hook: $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_mom.service $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service; \ fi \ elif [ -f $(top_srcdir)/contrib/init.d/pbs_mom ]; then \ - echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom"; \ - $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_mom $(DESTDIR)/etc/init.d/pbs_mom; \ + prefix=''; \ + if [ "`$(top_srcdir)/buildutils/determine.distro`" = sles ]; then \ + prefix=suse. ; \ + fi; \ + echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/$${prefix}pbs_mom $(DESTDIR)/etc/init.d/pbs_mom"; \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/$${prefix}pbs_mom $(DESTDIR)/etc/init.d/pbs_mom; \ fi; \ $(install_sh) -d -m 755 $(DESTDIR)/etc/ld.so.conf.d || :; \ if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ diff --git a/src/server/Makefile.am b/src/server/Makefile.am index 334b352c04..f96107011a 100644 --- a/src/server/Makefile.am +++ b/src/server/Makefile.am @@ -59,10 +59,14 @@ install-exec-hook: if [ -f $(top_srcdir)/contrib/systemd/pbs_server.service ]; then \ echo "$(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service"; \ $(install_sh) -m 644 $(top_srcdir)/contrib/systemd/pbs_server.service $(DESTDIR)/usr/lib/systemd/system/pbs_server.service; \ - fi \ + fi; \ elif [ -f $(top_srcdir)/contrib/init.d/pbs_server ]; then \ - echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server"; \ - $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/pbs_server $(DESTDIR)/etc/init.d/pbs_server; \ + prefix=''; \ + if [ "`$(top_srcdir)/buildutils/determine.distro`" = sles ]; then \ + prefix=suse. ; \ + fi; \ + echo "$(install_sh) -m 755 $(top_srcdir)/contrib/init.d/$${prefix}pbs_server $(DESTDIR)/etc/init.d/pbs_server"; \ + $(install_sh) -m 755 $(top_srcdir)/contrib/init.d/$${prefix}pbs_server $(DESTDIR)/etc/init.d/pbs_server; \ fi; \ fi From f42254bfa17e3e3f82c139877b990b20ac5a1573 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 17 Oct 2016 16:31:09 -0600 Subject: [PATCH 032/312] TRQ-3816 - RPM install won't write server_name into mom config file --- buildutils/torque.adaptive.spec.el6 | 2 +- buildutils/torque.adaptive.spec.el7 | 2 +- buildutils/torque.adaptive.spec.sles12 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildutils/torque.adaptive.spec.el6 b/buildutils/torque.adaptive.spec.el6 index 3bce650dab..d75df79c86 100644 --- a/buildutils/torque.adaptive.spec.el6 +++ b/buildutils/torque.adaptive.spec.el6 @@ -595,7 +595,7 @@ done ## Configuration echo '__AC_HOSTNAME_NOT_SET__ np=__AC_PROCS_NOT_SET__' > \ %{buildroot}%{torque_sysconfdir}/server_priv/nodes -echo '$pbsserver __AC_HOSTNAME_NOT_SET__' > \ +echo '$logevent 225 # bitmap of which events to log' > \ %{buildroot}%{torque_sysconfdir}/mom_priv/config %{__mkdir_p} %{buildroot}/etc/profile.d diff --git a/buildutils/torque.adaptive.spec.el7 b/buildutils/torque.adaptive.spec.el7 index 4e1bfb09c7..fb290cf021 100644 --- a/buildutils/torque.adaptive.spec.el7 +++ b/buildutils/torque.adaptive.spec.el7 @@ -594,7 +594,7 @@ done ## Configuration echo '__AC_HOSTNAME_NOT_SET__ np=__AC_PROCS_NOT_SET__' > \ %{buildroot}%{torque_sysconfdir}/server_priv/nodes -echo '$pbsserver __AC_HOSTNAME_NOT_SET__' > \ +echo '$logevent 225 # bitmap of which events to log' > \ %{buildroot}%{torque_sysconfdir}/mom_priv/config %{__mkdir_p} %{buildroot}/etc/profile.d diff --git a/buildutils/torque.adaptive.spec.sles12 b/buildutils/torque.adaptive.spec.sles12 index 176897df63..8e191db721 100644 --- a/buildutils/torque.adaptive.spec.sles12 +++ b/buildutils/torque.adaptive.spec.sles12 @@ -594,7 +594,7 @@ done ## Configuration echo '__AC_HOSTNAME_NOT_SET__ np=__AC_PROCS_NOT_SET__' > \ %{buildroot}%{torque_sysconfdir}/server_priv/nodes -echo '$pbsserver __AC_HOSTNAME_NOT_SET__' > \ +echo '$logevent 225 # bitmap of which events to log' > \ %{buildroot}%{torque_sysconfdir}/mom_priv/config %{__mkdir_p} %{buildroot}/etc/profile.d From 531011942e04b17b771d3e64e2a93fbf870a9d3f Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 18 Oct 2016 08:56:57 -0600 Subject: [PATCH 033/312] TRQ-3806 - cgroup handling in torque-build --- buildutils/torque-build | 4 ++++ buildutils/torque.adaptive.spec.el6 | 7 ++++--- buildutils/torque.adaptive.spec.el7 | 9 +++++---- buildutils/torque.adaptive.spec.sles12 | 9 +++++---- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index e6cf5f16f7..fa71c500b3 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -222,7 +222,10 @@ echo "Started ${0}, version ${version}, release ${release}." \ if [ $cgroups -ne 0 ] then + cgroupsopts="--enable-cgroups" configure_options="$configure_options --enable-cgroups --with-hwloc-path=$hwloc_path" +else + cgroupsopts="--disable-cgroups" fi # set so child makefiles know this @@ -339,6 +342,7 @@ then --define "_specdir ${root_path}/buildutils/rpmbuild/SPECS" \ --define "_tmppath ${root_path}/buildutils/rpmbuild/tmp" \ --define "_hwlocpath $hwloc_path" \ + --define "_cgroupsopt ${cgroupsopts}" \ --define "version ${version}" \ --define "release ${release}.${distro}" \ --define "source_file ${source_file}" \ diff --git a/buildutils/torque.adaptive.spec.el6 b/buildutils/torque.adaptive.spec.el6 index d75df79c86..7fc84ab6c9 100644 --- a/buildutils/torque.adaptive.spec.el6 +++ b/buildutils/torque.adaptive.spec.el6 @@ -67,6 +67,10 @@ then \ CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ fi \ + if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS="${CONFFLAGS} %{_cgroupsopt}" \ + fi \ if grep -q -- '--oldincludedir' "${CONFFLAGS_FILE}" \ then \ CONFFLAGS="${CONFFLAGS} --oldincludedir=%{_oldincludedir}" \ @@ -455,9 +459,6 @@ Obsoletes: %{community_client_pkg} < %{version}-%{release} Provides: %{community_client_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_client_pkg}(x86-64) < %{version}-%{release} %endif -%if %{with cgroups} -Requires: %{breq_cgroups} -%endif Provides: %{community_top_pkg}-docs = %{version}-%{release} Obsoletes: %{community_top_pkg}-docs < %{version}-%{release} Requires: %{common_pkg} diff --git a/buildutils/torque.adaptive.spec.el7 b/buildutils/torque.adaptive.spec.el7 index fb290cf021..74dd1b2065 100644 --- a/buildutils/torque.adaptive.spec.el7 +++ b/buildutils/torque.adaptive.spec.el7 @@ -67,6 +67,10 @@ then \ CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ fi \ + if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS="${CONFFLAGS} %{_cgroupsopt}" \ + fi \ if grep -q -- '--oldincludedir' "${CONFFLAGS_FILE}" \ then \ CONFFLAGS="${CONFFLAGS} --oldincludedir=%{_oldincludedir}" \ @@ -333,7 +337,7 @@ %define breq_munge %{?with_munge:munge-devel} %define breq_pam %{?with_pam:pam-devel} %define breq_scp %{?with_scp:/usr/bin/scp} -%define breq_cgroups %{?with_cgroups:libcgroup} %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} +%define breq_cgroups %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} # Missing: # libcpuset -> libcpuset.so.* @@ -455,9 +459,6 @@ Obsoletes: %{community_client_pkg} < %{version}-%{release} Provides: %{community_client_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_client_pkg}(x86-64) < %{version}-%{release} %endif -%if %{with cgroups} -Requires: %{breq_cgroups} -%endif Provides: %{community_top_pkg}-docs = %{version}-%{release} Obsoletes: %{community_top_pkg}-docs < %{version}-%{release} Requires: %{common_pkg} diff --git a/buildutils/torque.adaptive.spec.sles12 b/buildutils/torque.adaptive.spec.sles12 index 8e191db721..f6277f48d8 100644 --- a/buildutils/torque.adaptive.spec.sles12 +++ b/buildutils/torque.adaptive.spec.sles12 @@ -67,6 +67,10 @@ then \ CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ fi \ + if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS="${CONFFLAGS} %{_cgroupsopt}" \ + fi \ if grep -q -- '--oldincludedir' "${CONFFLAGS_FILE}" \ then \ CONFFLAGS="${CONFFLAGS} --oldincludedir=%{_oldincludedir}" \ @@ -333,7 +337,7 @@ %define breq_munge %{?with_munge:munge-devel} %define breq_pam %{?with_pam:pam-devel} %define breq_scp %{?with_scp:/usr/bin/scp} -%define breq_cgroups %{?with_cgroups:libcgroup1} %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} +%define breq_cgroups %{?with_cgroups:libcgroup-tools} %{?with_cgroups:hwloc >= 1.9.1} # Missing: # libcpuset -> libcpuset.so.* @@ -455,9 +459,6 @@ Obsoletes: %{community_client_pkg} < %{version}-%{release} Provides: %{community_client_pkg}(x86-64) = %{version}-%{release} Obsoletes: %{community_client_pkg}(x86-64) < %{version}-%{release} %endif -%if %{with cgroups} -Requires: %{breq_cgroups} -%endif Provides: %{community_top_pkg}-docs = %{version}-%{release} Obsoletes: %{community_top_pkg}-docs < %{version}-%{release} Requires: %{common_pkg} From bc7879e99e04874f035092c1129529f5b0a8f893 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 18 Oct 2016 09:32:01 -0600 Subject: [PATCH 034/312] Fix sles12 spec file --- buildutils/torque.adaptive.spec.sles12 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque.adaptive.spec.sles12 b/buildutils/torque.adaptive.spec.sles12 index f6277f48d8..74dd1b2065 100644 --- a/buildutils/torque.adaptive.spec.sles12 +++ b/buildutils/torque.adaptive.spec.sles12 @@ -67,7 +67,7 @@ then \ CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ fi \ - if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ + if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ then \ CONFFLAGS="${CONFFLAGS} %{_cgroupsopt}" \ fi \ From cf9459956fa0c5da9b0f669861a682b27d4c406b Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Tue, 18 Oct 2016 09:31:21 -0700 Subject: [PATCH 035/312] I have updated the configure logic to reject anything other than the PMIx v1.1.5 (or something further along the v1 series, should we release something) as that is the only release out there today. Future releases can be supported with the appropriate #if checks for PMIX_VERSION in the code. I found a race condition in the PMIx integration that registers nspaces and clients. The PMIx APIs for that purpose are non-blocking, but they must complete prior to you fork/exec'ing any local clients. Otherwise, a connection request that arrives at the PMIx server prior to completing the registration functions will be rejected. I don't know your threading model enough to figure out how to implement the wait function, so I left some comments (they start with "RHC" so you can search for them) that explain what needs to be done. Hopefully, those will be clear enough to get you thru the problem. If not, feel free to holler. --- configure.ac | 9 ++-- src/resmom/mom_job_func.c | 83 ++++++++++++++++++++++++---------- src/resmom/pmix_interface.c | 90 ++++++------------------------------- 3 files changed, 78 insertions(+), 104 deletions(-) diff --git a/configure.ac b/configure.ac index c57e714c7e..eaddd247c3 100644 --- a/configure.ac +++ b/configure.ac @@ -1401,7 +1401,7 @@ AS_IF([test -z "$with_pmix" || test "$with_pmix" = "no"], #endif ], [])], [AC_MSG_RESULT([found]) - pmix_version=3X + pmix_version=3 pmix_version_found=1], [AC_MSG_RESULT([not found])])]) @@ -1414,7 +1414,7 @@ AS_IF([test -z "$with_pmix" || test "$with_pmix" = "no"], #endif ], [])], [AC_MSG_RESULT([found]) - pmix_version=2X + pmix_version=2 pmix_version_found=1], [AC_MSG_RESULT([not found])])]) @@ -1427,11 +1427,11 @@ AS_IF([test -z "$with_pmix" || test "$with_pmix" = "no"], #endif ], [])], [AC_MSG_RESULT([found]) - pmix_version=1X + pmix_version=1 pmix_version_found=1], [AC_MSG_RESULT([not found])])]) - AS_IF([test "x$pmix_version" = "x"], + AS_IF([test "x$pmix_version" = "x" || test "$pmix_version" != "1"], [AC_MSG_WARN([PMIx support requested, but version]) AC_MSG_WARN([information of the external lib could not]) AC_MSG_WARN([be detected]) @@ -1449,6 +1449,7 @@ if test "${build_mom}" = "yes" ; then if test "${pmix_happy}" = "yes"; then AC_MSG_RESULT([yes - version: $pmix_version]) AC_DEFINE([ENABLE_PMIX], 1, [Define to enable pmix support]) + AC_DEFINE([PMIX_VERSION], [$pmix_version], [The PMIx version we are building against]) CPPFLAGS="-I$pmix_ext_install_dir/include $CPPFLAGS" LDFLAGS="-L$pmix_ext_install_dir/lib $LDFLAGS" MOMLIBS="$MOMLIBS -lpmix" diff --git a/src/resmom/mom_job_func.c b/src/resmom/mom_job_func.c index ee48165393..b037571ba9 100644 --- a/src/resmom/mom_job_func.c +++ b/src/resmom/mom_job_func.c @@ -316,7 +316,7 @@ int remtree( { sprintf(log_buffer, "unlink failed on %s", namebuf); log_err(errno, __func__, log_buffer); - + rtnv = -1; } else if (LOGLEVEL >= 7) @@ -383,7 +383,7 @@ int conn_qsub( { pbs_net_t hostaddr; - + int s; int local_errno = 0; int flags; @@ -515,7 +515,7 @@ void deregister_jobs_nspace( if (pos != std::string::npos) nspace.erase(pos); - PMIx_server_deregister_nspace(nspace.c_str(), NULL, NULL); + PMIx_server_deregister_nspace(nspace.c_str()); if (LOGLEVEL >= 6) { @@ -695,18 +695,18 @@ void remove_tmpdir_file( sprintf(log_buffer, "remtree failed: %s", strerror(errno)); log_err(errno, __func__, log_buffer); } - + setuid_ext(pbsuser, TRUE); setegid(pbsgroup); } - - if ((rc != 0) && + + if ((rc != 0) && (LOGLEVEL >= 5)) { sprintf(log_buffer, "recursive remove of job transient tmpdir %s failed", namebuf); - + log_err(errno, "recursive (r)rmdir", log_buffer); } } @@ -728,8 +728,8 @@ void *delete_job_files( if (thread_unlink_calls == true) { - /* this algorithm needs to make sure the - thread for delete_job_files posts to the + /* this algorithm needs to make sure the + thread for delete_job_files posts to the semaphore before it tries to lock the delete_job_files_mutex. posting to delete_job_files_sem lets other processes know there are threads still @@ -740,7 +740,7 @@ void *delete_job_files( { log_err(-1, __func__, "failed to post delete_job_files_sem"); } - + pthread_mutex_lock(&delete_job_files_mutex); } #ifdef PENABLE_LINUX26_CPUSETS @@ -762,10 +762,10 @@ void *delete_job_files( /* delete the node file and gpu file */ sprintf(namebuf,"%s/%s", path_aux, jfdi->jobid); unlink_ext(namebuf); - + sprintf(namebuf, "%s/%sgpu", path_aux, jfdi->jobid); unlink_ext(namebuf); - + sprintf(namebuf, "%s/%smic", path_aux, jfdi->jobid); unlink_ext(namebuf); @@ -852,9 +852,9 @@ void *delete_job_files( if (thread_unlink_calls == true) { - /* decrement the delte_job_files_sem so + /* decrement the delte_job_files_sem so other threads know this job is done - cleaning up + cleaning up */ sem_wait(delete_job_files_sem); pthread_mutex_unlock(&delete_job_files_mutex); @@ -906,7 +906,7 @@ void remove_from_exiting_list( exiting_job_list.erase(exiting_job_list.begin() + i); break; } - } + } } /* END remove_from_exiting_list() */ @@ -1102,7 +1102,7 @@ job *mom_find_job( if ((pos = jid.find_last_of('.')) != std::string::npos) { jid.erase(pos); - + pj = mom_find_job_by_raw_string(jid); } } @@ -1163,7 +1163,7 @@ bool am_i_mother_superior( { bool mother_superior = ((pjob.ji_nodeid == 0) && ((pjob.ji_qs.ji_svrflags & JOB_SVFLG_HERE) != 0)); - + return(mother_superior); } @@ -1211,6 +1211,24 @@ void free_info_array( uint32_t local_arch = 0xFFFFFFFF; +/* RHC: The register_nspace function is non-blocking and + * you have to wait for it to complete before moving on. + * The register_client function is also non-blocking and + * also must complete before you start to launch local + * procs, but it can run in parallel with the register_nspace + * function as the two are separate. So you may need a struct + * that contains whatever info you need to free in addition + * to a volatile flag to tell you when the function is complete */ +static void fincb( + + pmix_status_t status, + void *cbdata) + + { + volatile bool *active = (volatile bool*)cbdata; + *active = false; + } + void register_jobs_nspace( job *pjob, @@ -1226,8 +1244,9 @@ void register_jobs_nspace( std::string node_list; pmix_status_t rc; std::string pmix_jobid(pjob->ji_qs.ji_jobid); - - // Get some needed info about ranks / local execution slots + volatile bool active; + + // Get some needed info about ranks / local execution slots for (int i = 0; i < pjob->ji_numvnod; i++) { if (prev_index != pjob->ji_vnods[i].vn_host->hn_node) @@ -1275,7 +1294,7 @@ void register_jobs_nspace( strcpy(pmi_array[attr_index].key, PMIX_JOB_SIZE); pmi_array[attr_index].value.type = PMIX_UINT32; pmi_array[attr_index].value.data.uint32 = pjob->ji_numvnod; - + // Max procs for the job is also the same attr_index++; strcpy(pmi_array[attr_index].key, PMIX_MAX_PROCS); @@ -1287,7 +1306,7 @@ void register_jobs_nspace( strcpy(pmi_array[attr_index].key, PMIX_NODE_SIZE); pmi_array[attr_index].value.type = PMIX_UINT32; pmi_array[attr_index].value.data.uint32 = es; - + // Local size is the same as the number for this node attr_index++; strcpy(pmi_array[attr_index].key, PMIX_LOCAL_SIZE); @@ -1342,7 +1361,7 @@ void register_jobs_nspace( std::string range; std::string cpu_list(pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str); find_range_in_cpuset_string(cpu_list, range); - + attr_index++; strcpy(pmi_array[attr_index].key, PMIX_CPUSET); pmi_array[attr_index].value.type = PMIX_STRING; @@ -1396,6 +1415,9 @@ void register_jobs_nspace( if (pos != std::string::npos) pmix_jobid.erase(pos); + /* RHC: You can wait here for this function to complete, + * or you can put a wait at the end of this entire function + * as I have shown below */ if ((rc = PMIx_server_register_nspace(pmix_jobid.c_str(), es, pmi_array, @@ -1410,7 +1432,7 @@ void register_jobs_nspace( PMIx_Error_string(rc)); log_err(-1, __func__, log_buffer);*/ } - else + else { if (LOGLEVEL >= 3) { @@ -1425,6 +1447,10 @@ void register_jobs_nspace( strcpy(p.nspace, pmix_jobid.c_str()); p.rank = 0; + /* RHC: I would advise not waiting for this function to complete on + * each local client as that would be unnecessarily slow. I would instead + * suggest just adding a wait at the end of the entire procedure, as + * illustrated below */ rc = PMIx_server_register_client(&p, pjob->ji_qs.ji_un.ji_momt.ji_exuid, pjob->ji_qs.ji_un.ji_momt.ji_exgid, @@ -1448,8 +1474,12 @@ void register_jobs_nspace( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); } } - + // Peer-level information - do we provide this at this time? + + /* RHC: Yes, you need to provide this info for every process in the + * job or we will hang */ + // rank // appnum // application leader @@ -1463,6 +1493,11 @@ void register_jobs_nspace( // spawned - true if launched via a dynamic spawn // temporary dir for this process + active = true; + while (active) { + usleep(10); + } + } // END register_jobs_nspace() #endif diff --git a/src/resmom/pmix_interface.c b/src/resmom/pmix_interface.c index 2c0d12ffca..0da04c43c4 100644 --- a/src/resmom/pmix_interface.c +++ b/src/resmom/pmix_interface.c @@ -28,9 +28,7 @@ bool pmix_spawn_timed_out; pmix_status_t pmix_client_connect( const pmix_proc_t *proc, - void *svr_obj, - pmix_op_cbfunc_t cbfunc, - void *cbdata) + void *svr_obj) { pmix_status_t rc = PMIX_SUCCESS; @@ -50,9 +48,6 @@ pmix_status_t pmix_client_connect( else rc = PMIX_ERR_NOT_FOUND; - if (cbfunc != NULL) - cbfunc(rc, cbdata); - return(rc); } // END pmi_client_connect() @@ -160,7 +155,7 @@ bool check_and_record_fence( pmix_operation fence(pjob, procs, nprocs, info, cbfunc, cbdata); pending_fences[jobid] = fence; } - + return(pending_fences[jobid].mark_reported(mom_host)); } // END check_and_record_fence() @@ -491,7 +486,7 @@ pmix_status_t pmix_server_spawn( time_t start_time; time_t global_tcp_timeout = 0; //bool notify = false; - + memset(&timeout, 0, sizeof(timeout)); if (pjob == NULL) @@ -512,7 +507,7 @@ pmix_status_t pmix_server_spawn( notify = true; } */ } - + if (timeout.tv_sec != 0) { start_time = time(NULL); @@ -677,12 +672,10 @@ pmix_status_t pmix_server_disconnect( pmix_status_t pmix_server_register_events( - pmix_status_t *codes, - size_t ncodes, - const pmix_info_t info[], - size_t ninfo, - pmix_op_cbfunc_t cbfunc, - void *cbdata) + const pmix_info_t info[], + size_t ninfo, + pmix_op_cbfunc_t cbfunc, + void *cbdata) { return(PMIX_ERR_NOT_SUPPORTED); @@ -692,66 +685,16 @@ pmix_status_t pmix_server_register_events( pmix_status_t pmix_server_deregister_events( - pmix_status_t *codes, - size_t ncodes, - pmix_op_cbfunc_t cbfunc, - void *cbdata) + const pmix_info_t info[], + size_t ninfo, + pmix_op_cbfunc_t cbfunc, + void *cbdata) { return(PMIX_ERR_NOT_SUPPORTED); } // END pmix_server_deregister_event() -pmix_status_t pmix_server_notify_event( - - pmix_status_t code, - const pmix_proc_t *source, - pmix_data_range_t range, - pmix_info_t info[], - size_t ninfo, - pmix_op_cbfunc_t cbfunc, - void *cbdata) - - { - return(PMIX_ERR_NOT_SUPPORTED); - } // END pmix_server_notify_event() - -pmix_status_t pmix_server_query( - - pmix_proc_t *proct, - pmix_query_t *queries, - size_t nqueries, - pmix_info_cbfunc_t cbfunc, - void *cbdata) - - { - return(PMIX_ERR_NOT_SUPPORTED); - } // END pmix_server_query() - -void pmix_server_tool_connection( - - pmix_info_t *info, - size_t ninfo, - pmix_tool_connection_cbfunc_t cbfunc, - void *cbdata) - - { - } // END pmix_server_tool_connection() - -void pmix_server_log( - - const pmix_proc_t *client, - const pmix_info_t data[], - size_t ndata, - const pmix_info_t directives[], - size_t ndirectives, - pmix_op_cbfunc_t cbfunc, - void *cbdata) - - { - } - - -pmix_server_module_t psm = +pmix_server_module_t psm = { pmix_client_connect, pmix_client_finalize, @@ -765,12 +708,7 @@ pmix_server_module_t psm = pmix_server_connect, pmix_server_disconnect, pmix_server_register_events, - pmix_server_deregister_events, - pmix_server_notify_event, - NULL, // instructs pmix server to listen by it's lonesome - pmix_server_query, - pmix_server_tool_connection, - pmix_server_log + pmix_server_deregister_events }; From 529e34a7ce7ce4043c23077ce4f19b5d1dfb5c96 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Sun, 2 Oct 2016 21:08:24 +0200 Subject: [PATCH 036/312] Add zlib support to tracejob tracejob can now read both plain text and gzip compressed files. --- .gitignore | 1 + configure.ac | 4 ++++ src/tools/tracejob.c | 14 +++++++------- src/tools/tracejob.h | 4 ++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 4339c82d9e..bc690be90a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ test_uut .deps .dirstamp .libs +*~ *.m4 *.log *.trs diff --git a/configure.ac b/configure.ac index eaddd247c3..bf86fb23a4 100644 --- a/configure.ac +++ b/configure.ac @@ -800,6 +800,10 @@ AC_CHECK_LIB(${xmlLib}, xmlGetProp, [AC_MSG_ERROR([TORQUE needs libxml2-devel in order to build]) ]) +dnl find zlib +AC_CHECK_LIB(z, gzopen, + [], + [AC_MSG_ERROR([TORQUE needs zlib-devel in order to build]) ]) dnl ########################################### diff --git a/src/tools/tracejob.c b/src/tools/tracejob.c index 481c2996ef..f64805bf40 100644 --- a/src/tools/tracejob.c +++ b/src/tools/tracejob.c @@ -116,7 +116,7 @@ int main( { /* Array for the log entries for the specified job */ - FILE *fp; + gzFile fp; int i, j; int file_count; char *filenames[MAX_LOG_FILES_PER_DAY]; /* full path of logfiles to read */ @@ -333,7 +333,7 @@ int main( for (; file_count > 0; file_count--) { - if ((fp = fopen(filenames[file_count-1], "r")) == NULL) + if ((fp = gzopen(filenames[file_count-1], "r")) == NULL) { if (verbosity >= 1) perror(filenames[file_count-1]); @@ -341,7 +341,7 @@ int main( continue; } - if (parse_log(fp, argv[opt], j) < 0) + if (parse_log(&fp, argv[opt], j) < 0) { /* no valid entries located in file */ @@ -357,7 +357,7 @@ int main( filenames[file_count-1]); } - fclose(fp); + gzclose(fp); free(filenames[file_count-1]); } /* end of for file_count */ } @@ -414,7 +414,7 @@ int main( int parse_log( - FILE *fp, /* I */ + gzFile *fp, /* I */ char *job, /* I */ int ind) /* I */ @@ -434,7 +434,7 @@ int parse_log( tms.tm_isdst = -1; /* mktime() will attempt to figure it out */ - while (fgets(buf, sizeof(buf), fp) != NULL) + while (gzgets(*fp, buf, sizeof(buf)) != NULL) { lineno++; j++; @@ -589,7 +589,7 @@ int parse_log( logcount++; } - } /* END while (fgets(buf,sizeof(buf),fp) != NULL) */ + } /* END while (gzgets(*fp, buf, sizeof(buf)) != NULL) */ if (logcount == 0) { diff --git a/src/tools/tracejob.h b/src/tools/tracejob.h index 42ff071426..e82de28ec4 100644 --- a/src/tools/tracejob.h +++ b/src/tools/tracejob.h @@ -82,7 +82,7 @@ #define TRACEJOB_H #include /* time_t, struct tm */ -#include /* FILE */ +#include /* Symbolic constants */ @@ -158,7 +158,7 @@ unsigned no_print: /* prototypes */ int sort_by_date(const void *v1, const void *v2); -int parse_log(FILE *, char *, int); +int parse_log(gzFile *, char *, int); char *strip_path(char *path); void free_log_entry(struct log_entry *lg); void line_wrap(char *line, int start, int end); From 79f2ba80bbd3a539678b3564d83f60cea01d3add Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Tue, 4 Oct 2016 15:26:07 +0200 Subject: [PATCH 037/312] Get rid of global variables in tracejob Instead of global variables (which make it impossible to do unit tests), we put them in a struct and pass the struct to all functions needed. This commit also fixes a couple of memory leaks. --- src/tools/tracejob.c | 179 +++++++++++++++++++++---------------------- src/tools/tracejob.h | 23 +++++- 2 files changed, 106 insertions(+), 96 deletions(-) diff --git a/src/tools/tracejob.c b/src/tools/tracejob.c index f64805bf40..a861e38348 100644 --- a/src/tools/tracejob.c +++ b/src/tools/tracejob.c @@ -94,21 +94,6 @@ #include "tracejob.h" -/* path from pbs home to the log files */ - -const char *mid_path[] = - { - "server_priv/accounting", - "server_logs", - "mom_logs", - "sched_logs" - }; - -struct log_entry *log_lines; -int ll_cur_amm; -int ll_max_amm; - - int main( int argc, @@ -117,7 +102,7 @@ int main( { /* Array for the log entries for the specified job */ gzFile fp; - int i, j; + unsigned int i, j; int file_count; char *filenames[MAX_LOG_FILES_PER_DAY]; /* full path of logfiles to read */ @@ -125,7 +110,7 @@ int main( time_t t, t_save; signed char c; char *prefix_path = NULL; - int number_of_days = 1; + unsigned int number_of_days = 1; char *endp; short error = 0; int opt; @@ -136,6 +121,7 @@ int main( int event_type; char filter_excessive = 0; int excessive_count; + struct log_array log = {}; #if defined(FILTER_EXCESSIVE) filter_excessive = 1; @@ -218,7 +204,7 @@ int main( case 'n': - number_of_days = strtol(optarg, &endp, 10); + number_of_days = strtoul(optarg, &endp, 10); if (*endp != '\0') error = 1; @@ -321,10 +307,10 @@ int main( continue; file_count = log_path(prefix_path, j, tm_ptr, filenames); - + /* there can be multiple server and mom log files per day */ /* traverse filenames until we have got them all */ - + if (file_count < 0) { printf("Error getting file names\n"); @@ -338,10 +324,11 @@ int main( if (verbosity >= 1) perror(filenames[file_count-1]); + free(filenames[file_count-1]); continue; } - if (parse_log(&fp, argv[opt], j) < 0) + if (parse_log(&fp, argv[opt], j, &log) < 0) { /* no valid entries located in file */ @@ -364,34 +351,38 @@ int main( } /* END for (i) */ if (filter_excessive) - filter_excess(excessive_count); + filter_excess(excessive_count, &log); - qsort(log_lines, ll_cur_amm, sizeof(struct log_entry), sort_by_date); + qsort(log.log_lines, log.ll_cur_amm, sizeof(struct log_entry), sort_by_date); - if (ll_cur_amm != 0) + if (log.ll_cur_amm != 0) { printf("\nJob: %s\n\n", - log_lines[0].name); + log.log_lines[0].name); } - for (i = 0;i < ll_cur_amm;i++) + for (i = 0;i < log.ll_cur_amm;i++) { - if (log_lines[i].log_file == 'A') + if (log.log_lines[i].log_file == 'A') event_type = 0; else - event_type = strtol(log_lines[i].event, &endp, 16); + event_type = strtol(log.log_lines[i].event, &endp, 16); - if (!(log_filter & event_type) && !(log_lines[i].no_print)) + if (!(log_filter & event_type) && !(log.log_lines[i].no_print)) { printf("%-20s %-5c", - log_lines[i].date, - log_lines[i].log_file); + log.log_lines[i].date, + log.log_lines[i].log_file); - line_wrap(log_lines[i].msg, 26, wrap); + line_wrap(log.log_lines[i].msg, 26, wrap); } } } /* END for (opt) */ + for (i = 0;i < log.ll_cur_amm;i++) + free_log_entry(&log.log_lines[i]); + free(log.log_lines); + return(0); } /* END main() */ @@ -406,9 +397,9 @@ int main( * fp - the log file * job - the name of the job * ind - which log file - index in enum index + * log - the log_array where to save the entries * * returns nothing - * modifies global variables: loglines, ll_cur_amm, ll_max_amm * */ @@ -416,7 +407,8 @@ int parse_log( gzFile *fp, /* I */ char *job, /* I */ - int ind) /* I */ + int ind, /* I */ + struct log_array *log) /* I */ { @@ -445,24 +437,24 @@ int parse_log( pa = buf; memset(&tmp, 0, sizeof(struct log_entry)); - for(field_count = 0; (pa != NULL) && (field_count <= FLD_MSG); field_count++) + for(field_count = 0; (pa != NULL) && (field_count <= FLD_MSG); field_count++) { /* instead of using strtok every time, conditionally advance the pa (the field pointer) * on semicolons. This prevents data from getting cut out of messages with semicolons in * them */ - if(field_count < FLD_MSG) + if(field_count < FLD_MSG) { if((pe = strchr(pa, ';'))) *pe = '\0'; - } - else + } + else { pe = NULL; } - switch (field_count) - + switch (field_count) + { case FLD_DATE: @@ -473,34 +465,34 @@ int parse_log( break; case FLD_EVENT: - + tmp.event = pa; - + break; case FLD_OBJ: - + tmp.obj = pa; - + break; case FLD_TYPE: - + tmp.type = pa; - - + + break; case FLD_NAME: - + tmp.name = pa; - + break; case FLD_MSG: - + tmp.msg = pa; - + break; } @@ -515,77 +507,77 @@ int parse_log( !strncmp(job, tmp.name, strlen(job)) && !isdigit(tmp.name[strlen(job)])) { - if (ll_cur_amm >= ll_max_amm) - alloc_more_space(); + if (log->ll_cur_amm >= log->ll_max_amm) + alloc_more_space(log); - free_log_entry(&log_lines[ll_cur_amm]); + free_log_entry(&log->log_lines[log->ll_cur_amm]); if (tmp.date != NULL) { - log_lines[ll_cur_amm].date = strdup(tmp.date); + log->log_lines[log->ll_cur_amm].date = strdup(tmp.date); if (sscanf(tmp.date, "%d/%d/%d %d:%d:%d", &tms.tm_mon, &tms.tm_mday, &tms.tm_year, &tms.tm_hour, &tms.tm_min, &tms.tm_sec) != 6) - log_lines[ll_cur_amm].date_time = -1; /* error in date field */ + log->log_lines[log->ll_cur_amm].date_time = -1; /* error in date field */ else { if (tms.tm_year > 1900) tms.tm_year -= 1900; - log_lines[ll_cur_amm].date_time = mktime(&tms); + log->log_lines[log->ll_cur_amm].date_time = mktime(&tms); } } if (tmp.event != NULL) - log_lines[ll_cur_amm].event = strdup(tmp.event); + log->log_lines[log->ll_cur_amm].event = strdup(tmp.event); else - log_lines[ll_cur_amm].event = none; + log->log_lines[log->ll_cur_amm].event = none; if (tmp.obj != NULL) - log_lines[ll_cur_amm].obj = strdup(tmp.obj); + log->log_lines[log->ll_cur_amm].obj = strdup(tmp.obj); else - log_lines[ll_cur_amm].obj = none; + log->log_lines[log->ll_cur_amm].obj = none; if (tmp.type != NULL) - log_lines[ll_cur_amm].type = strdup(tmp.type); + log->log_lines[log->ll_cur_amm].type = strdup(tmp.type); else - log_lines[ll_cur_amm].type = none; + log->log_lines[log->ll_cur_amm].type = none; if (tmp.name != NULL) - log_lines[ll_cur_amm].name = strdup(tmp.name); + log->log_lines[log->ll_cur_amm].name = strdup(tmp.name); else - log_lines[ll_cur_amm].name = none; + log->log_lines[log->ll_cur_amm].name = none; if (tmp.msg != NULL) - log_lines[ll_cur_amm].msg = strdup(tmp.msg); + log->log_lines[log->ll_cur_amm].msg = strdup(tmp.msg); else - log_lines[ll_cur_amm].msg = none; + log->log_lines[log->ll_cur_amm].msg = none; switch (ind) { case IND_SERVER: - log_lines[ll_cur_amm].log_file = 'S'; + log->log_lines[log->ll_cur_amm].log_file = 'S'; break; case IND_SCHED: - log_lines[ll_cur_amm].log_file = 'L'; + log->log_lines[log->ll_cur_amm].log_file = 'L'; break; case IND_ACCT: - log_lines[ll_cur_amm].log_file = 'A'; + log->log_lines[log->ll_cur_amm].log_file = 'A'; break; case IND_MOM: - log_lines[ll_cur_amm].log_file = 'M'; + log->log_lines[log->ll_cur_amm].log_file = 'M'; break; default: - log_lines[ll_cur_amm].log_file = 'U'; /* undefined */ + log->log_lines[log->ll_cur_amm].log_file = 'U'; /* undefined */ } - log_lines[ll_cur_amm].lineno = lineno; + log->log_lines[log->ll_cur_amm].lineno = lineno; - ll_cur_amm++; + log->ll_cur_amm++; logcount++; } @@ -894,24 +886,26 @@ int get_cols(void) * * alloc_space - double the allocation of current log entires * + * log - the log_array where to save the entries + * */ void -alloc_more_space(void) +alloc_more_space(struct log_array *log) { - int old_amm = ll_max_amm; + int old_amm = log->ll_max_amm; - if (ll_max_amm == 0) - ll_max_amm = DEFAULT_LOG_LINES; + if (log->ll_max_amm == 0) + log->ll_max_amm = DEFAULT_LOG_LINES; else - ll_max_amm *= 2; + log->ll_max_amm *= 2; - if ((log_lines = (struct log_entry *)realloc(log_lines, ll_max_amm * sizeof(struct log_entry))) == NULL) + if ((log->log_lines = (struct log_entry *)realloc(log->log_lines, log->ll_max_amm * sizeof(struct log_entry))) == NULL) { perror("Error allocating memory"); exit(1); } - memset(&log_lines[old_amm], 0, (ll_max_amm - old_amm) * sizeof(struct log_entry)); + memset(&log->log_lines[old_amm], 0, (log->ll_max_amm - old_amm) * sizeof(struct log_entry)); } @@ -924,27 +918,28 @@ alloc_more_space(void) * the message threshold * * threshold - if the number of messages exceeds this, don't print them + * log - the log_array where to save the entries * * returns nothing * * NOTE: log_lines array will be sorted in place */ -void filter_excess(int threshold) +void filter_excess(int threshold, struct log_array *log) { int cur_count = 1; char *msg; - int i; - int j = 0; + unsigned int i; + unsigned int j = 0; - if (ll_cur_amm) + if (log->ll_cur_amm) { - qsort(log_lines, ll_cur_amm, sizeof(struct log_entry), sort_by_message); - msg = log_lines[0].msg; + qsort(log->log_lines, log->ll_cur_amm, sizeof(struct log_entry), sort_by_message); + msg = log->log_lines[0].msg; - for (i = 1; i < ll_cur_amm; i++) + for (i = 1; i < log->ll_cur_amm; i++) { - if (strcmp(log_lines[i].msg, msg) == 0) + if (strcmp(log->log_lines[i].msg, msg) == 0) cur_count++; else { @@ -954,13 +949,13 @@ void filter_excess(int threshold) j++; for (; j < i; j++) - log_lines[j].no_print = 1; + log->log_lines[j].no_print = 1; } j = i; cur_count = 1; - msg = log_lines[i].msg; + msg = log->log_lines[i].msg; } } @@ -969,7 +964,7 @@ void filter_excess(int threshold) j++; for (; j < i; j++) - log_lines[j].no_print = 1; + log->log_lines[j].no_print = 1; } } } diff --git a/src/tools/tracejob.h b/src/tools/tracejob.h index e82de28ec4..48d7ac958b 100644 --- a/src/tools/tracejob.h +++ b/src/tools/tracejob.h @@ -138,7 +138,6 @@ enum field }; /* A PBS log entry */ - struct log_entry { char *date; /* date of log entry */ @@ -156,15 +155,31 @@ unsigned no_print: /* A=accounting S=server M=Mom L=Scheduler */ }; +/* path from pbs home to the log files */ +const char *mid_path[] = + { + "server_priv/accounting", + "server_logs", + "mom_logs", + "sched_logs" + }; + +struct log_array + { + struct log_entry *log_lines; + unsigned int ll_cur_amm; + unsigned int ll_max_amm; + }; + /* prototypes */ int sort_by_date(const void *v1, const void *v2); -int parse_log(gzFile *, char *, int); +int parse_log(gzFile *, char *, int, struct log_array *); char *strip_path(char *path); void free_log_entry(struct log_entry *lg); void line_wrap(char *line, int start, int end); int log_path(char *path, int index, struct tm *tm_ptr, char *filenames[]); -void alloc_more_space(); -void filter_excess(int threshold); +void alloc_more_space(struct log_array *); +void filter_excess(int threshold, struct log_array *); int sort_by_message(const void *v1, const void *v2); int get_cols(void); From 9dc7512ab04a2a328e8235ee0ae0120e4c55f9a4 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Tue, 4 Oct 2016 16:05:00 +0200 Subject: [PATCH 038/312] Add unit tests for tracejob --- .../test/tracejob/server_logs/20160928.gz | Bin 0 -> 7515 bytes src/tools/test/tracejob/server_logs/20161003 | 9269 +++++++++++++++++ src/tools/test/tracejob/test_tracejob.c | 112 +- 3 files changed, 9375 insertions(+), 6 deletions(-) create mode 100644 src/tools/test/tracejob/server_logs/20160928.gz create mode 100644 src/tools/test/tracejob/server_logs/20161003 diff --git a/src/tools/test/tracejob/server_logs/20160928.gz b/src/tools/test/tracejob/server_logs/20160928.gz new file mode 100644 index 0000000000000000000000000000000000000000..51b4e04feccba922d3e995037bfabaa17c528cd5 GIT binary patch literal 7515 zcmZ{oWmHse)b^1M0cjWnlnwy}q`OnP1{jcT1c{*pL_%O3y1Pqi2uVR?kdSn!k?v+_ z$#?ia&sy*M<^6Kjwa&TEUiaC1-)CLF%@m7|A0l^bjDaQa@|EyQF#*xH7{4vt-qW`x z+~r!nV#?um5$NvFD49dj7_RX4w>u0~RH<83IC{Feuq2VF`NJ-DNxL6w1z-pRh^->2 z9PHGa^Be*IkOO5u7MJ@US>KQ14AoLCDlMHBnvLncU!6sX_<6MbPIFIcU?0t@60sCE z=AUL-yH7gXbVu;sC+A%4M*OSw`0jeZ*zA{eu&6!n>vtx9lYl(*SnN_0o)5k_Dlua} z6xr1taPHg#CmC+H_f;bI+fEf)=1MG^FT|fYaJl$D>wbzgQ`+&XA9K(Q#NFL1QtLiV zAC`HVml)!mwci_%V$NZhDW%Og&BWNwKVNh?43$gAczav{PJ!lDR*J0`h+gVgqUKz0 z=D0^1mOS=DjsMt4PQAkfL4LYay*&PuqPT+6Fp4hYs%_YytsQ~w>};M%M6~Ste*e1Y z<`-)rau<{lP1+uXh*AvVr~u!M-fPM5Hn+R?`s^v*rsOW|?;cuq`g;bBhuoqL2i$XR zE|6I*A~F`ImWd8~AMHY<2h!3~66zv)4O_?)0~6zzj4hs&cl%Om|OUQ;hE5-(j# z0yz(f=P`(#9Tk+Bn(BwG1HTDtkpb%_%>x=Jbf@XCd0+(;y?TGB42zdi)W@OFV$cfkbNZH^+RVFB?v;D245iL|nnp|(!s&n8 zs7~E(t2)cRdof_&xp14NutLaA=2muNBrk7UyVwZA9f8Tg=##n8U68jD|O&}`>qS=o^-Jb3HaC-h`LCWmqgqKH2q~CO_i^lUJSmN z&&%V;y58Lr%KO~ck~X!!Yy7>Vb6@}O^&zok3clDg`NFwQ;34*Y%HW7^9yKX680f1b z_xuk+e4B5L^o{JU{Te<%(+g(;9Iq$@9a31$vJWEvoJOa2w}6r3dAk6(t3{#-ek0IQj57g{TAm7vJ4*1kAumEk>XqVY1j z8r3ji9YhX!*u{Gq2-77xGCu_GA#w?7!*+~J2&_)7x++ILek^GV@z5hdf_3bdJ2U?` zY^ClG+Ypl2UP@`kV&_3X3bKluiD3m1<%`lXDB3uo56cK|3=UK%_+`7873_+?^= z>FlS9h^FaMZpqTsac6yvry$)plmV}K+?;hp*_(d~f3B?KB7#Cp#SD32-K+A3O><6j z*Dmqkvu`Uomv@9uLG(MfuO5SPrVM@hzJVIA+OAu;f%D$qeJ&(Wx*rl!k5SMy&@*#E zvYYAAg&Jz@cIfW!-g|<}>E33R>eZEPnd#XR+l&6qgDXg#zP8z0&D;xYMC#gL10Bqu z_V%wyGeG0t@yKU87LbQ0Tj4iUj8X~L^dX}!o=>}Kt~5W5b8o&A^X1nX(tJik{T@i0u0H2DlkxYzt}TpRe~R3ntQdW8xy+R0 z;a5-M#}fRKDW5vFJ}@ZO4y-=F{*x0RVo41p9a@832079Eh3)I}AMGZQi)mGboDmvI zp3DclZvk`Md~3h`?zEY6b2CKh=dp12Y=7(b{Lu$}1h z{e8Z+t!z?ks0q=l`!nUsGH0B0^YI3)SZ?#q44&=U8+dj|n}3Dqro*7=F+E)-eF?zM z^P?w2%~C?)NO0W6Du2lR+4}GO-+gx?1F)S!S8ZMDES~!J%gX7|?H?w;rDnlPku`b3 z%zxSg=X{4w?@lHU#OXs?%zUy)L&_I-Wp4fiw~hGcP~8WJ+>O(m36jOXyLI@^&&cFQ zn%B15)$?k4knlt!wInXw6r-+GQjxk_nHl(-mX|1hYpOx;%$ph`*>ve-7YlvhR7No$ zC!b}qSL1Jw;Nb$QT}et&K2;*CI+k?JS8>%S$@PY+i^l)yd%ypu?-z0FANsCCosNE! zs_@K~M{=Z*(GQpl&%M||?Yv@q`{@E0Y zkrLLr_P35Zww}sU^9q)(TidKRQ!#qH4ld}Fxds@6#fTa=_FT-Dn_i5W;@d+`l(%AY zU#IdPxgOVJX-K&Q8MV6gH4M)9Z#EN&@u@c1HD*TcGlf=@l&uWjwG zLn$3sATBbY@9mLrQELfVQj6^#JIJ15r~op5se8B?Ld^{$K32ACb`vVBNK~;8^c_ha z_I(~3(OOXSmV4sU@VWmx-vQT)IJ3@iS9KrXAy?959$pxvNxMH||DFGSqzI$G8=t&* zZ#ZFy{^g-yfH-ZfWnD7c0$Q1*YC4D&c*I&~lx4VX$n2Aatsm#!rE|1}%QUMHu0dOU z1)kiz>bqR4Zi|LImhvv~BSzrpabbi!rp)eR3U?~Fc@-}*ffkDTZ2Tdv_9KcRFAjOw z%zh6aBA&Q{PFl32x7?ctgf5(9V-}26VOpi0nkKX0xzfssb}8p-J&a}o%ItaJM~yno z1Z$8MEewcb<(f-YrMgUXUX0+ZFrp;vR1r6YnkV;>L%)gc%Uu5|pD$EwcBTC>s*7B2 zs!eDzEUO(wCG<(1qgOeA&1+OIMI1J%tN^nwOXN7JLrd4M2xSk(5{zU`BSupVxLoXi zr&^8JQfcg;IuG8Fy%-YC5;Xg0ZCbX1MM)9*b*hW(vmzb~`IC7zX1aJxCDs~?qvU98 z+o$yeT~Wdi{MrQdM~bZZORi)Dszg)?p&j*T!IH`SPl@%!i(Q%32j~#llTDFO`p+g2 z_BMMhIh*13RnN>zR~vL)&iexfftI)v(tW6Y?xfLmx&%toOgmVBcshV*AaU16oZ_J# zts`FYi_`;pc0z9g6_-q~wL^v}=6jfsEI7yg<{U;3)tDLM|CkJc%CHw`{r>R!|62`B zuNq>CO^y}KlYYX~4Yu&Bt)AyfUCjfe^L@g^s$l__7Mm)%t`xsSS1!vL=L~3!pQT{d zX`w%^nV>kLLF<`EwHxL`U1gB0V!QP#2H6A66`Y$$Y}NY0HCIuh%w(8Zbvf;58Ni!s zbU5RgUGEbJ0Rvz<+FZA$Ao}U)DYMdAgo9LtFo&F!qbXnn$u6|@$xlBu5@@+b!mxc> zDgL^*aLvaC?2+!OmS9gM+Kbr*-QC3Cy%nZDKrGF;03D@nV?4gj;~hS0>b7}7S-`&5 z$71qmqW#J(dtbgdaB1C+9l3|q^mESa0~M$SLd4o^@Sn7Bb~|iLHMzz#0=towvrU2O z_>OiA+>_-N@;cXzm)E_NjiWCdJpLJi8`y+@UBk+Bs6f3h~-?rm0m-7ZDdH^OYq`F7{)nUCqXaz4{_*Q5NO? znBzOI9#^O$EvM*9GW((k^-p0Px}RUtKXi2hKD{VCpLuq)ARC#{pc%7%t=0G zLsGQ8*f9Y9S4Ug0K)k)=5$<3~ZlT;cp=s9DfCec@%ldRq+?@IFz4_=J- z*l<6YxAS9+$7IlN{SsKbH}Y&XdXh2F@iLvzh|tOp7smmaBL^&>r(1)}=g0xZ4F){? za?=Do{c0v&OzU6kCzS4wB*JpXvXfz}2QO&p4^DISa&r6})Wo6^PnX1@$;F$};iX!5 zoFyF{5iV(2io-^OL+=`771&2{;u{i!askKZTO1k=k;W*y!eRD4~_`X za)pefxU`fk&LE>i-l{lWIhI{#8x}7^z^6GIdcA2C#(bq$`56iMd)>4M<;NmI%J&0>!j!mjsKP;I0)rUcPBr*&7$_?rdLsan3=)n0mS zntv8*8X^z|*tvZ>m8Y3MHCIEqn_5}=o!3HRW@3TK{x-!wbDIEeLh#)v)k>gMCeEXx zGJIYS0ow3#@OUh7U0{iyEu%d}##yG{z_4pDPi_(+bc!M;@v8OUTg^e8D9 zG$S37RiZF*Sy(kx%GZP-G6hS4HK)V!#k9IB1MeAsN5$F(2cVN^8(clr(eKij+G;Xo zzViAuM=R-PG^Tb<<+-rFVp6)`zBY7VLge^-O&NjkWLlefpQl8*Yf*$qM}<{y5H-eN8!%> zhJmWHLcZVcPPgCsHC=z(rXE^GEY8c+7|+iiw%^5{zjd0#A+;j7sOxwfA=_bAExX8j z-jbCjWAW==l|d#54ClX8yR8YeAXCO&6vta^Tke2z0ytClcmN9I*Q!ERY zi#Px1h^>K+GNwiYS(e5tbgZNBLriiRivw6ZcFKHJ6IuA{ub0%c&e{BZ23!G2Ol z=oFvM`{cWR;{kBVUmXk@vXuHFNj#NolFfvbcTQMYnno|cZVctG2fBKR!C}CNJ&C8r zkUYcubN}lvMMsswV>h}FVovA8b6B^SiN&-p@VR~k^B?3TJ zJ)U|{IIN={ZQaoEZd{YrT(=6+qtOm}J@Vg;KAph@-J-*f*|$G0a_e(%a=fa1GLNKm zsNInRSf5z*dEMtn-u?ZY%Il9a{Cn0Phf)xr zvO>rSE*br%axnJFNH)ig<51mgutTmo)I4PDV~JMQIE+nG@$w${HN=wbxO}fqpmo-D ziarX1Dap%#^{i@l+zFp@AZ<6WKpe!1O#PiPE(aUl9Nx+|brzFc&(5RJ;KMGnDDlpo z#~=!qvXRjCj+$|YQgR-y0byFv`;<0r{c=Naj%J=nOp^C$K}``mn{<>?BL*)t?=RN7W~+29I0` zD0b~|O$wd*z4G=+s}>gq5}Rtzx`|RgkH?Y-)wu(vAhE9yZh5uubhLy5I zqsBdN&*L{QBr^QC2jTDbnb{8wN*Yksa94uTbItnfrd0gj4Vh>pv8~u~)b7W&(y#DP zT!}g$Pk5{v#!No_!BSljQ{c~n!K~O2Srpu@kH^kr~m!AOIJ~xz(4$8 zq>ZQFUSl${vu$N+1iNbHOm!PPJtmMNe)$zRxElR*x8RLM z7HAEJomnJ1KG`&rDpEL|fVge9T{l@n%IxFc0v1E9ma+vc{l?Nf$fDNUARTx&SIW{? z9T~iIZlQQlF&nt4T%Cx+SScY_8{}{J8*M5~^goLn^)Njv(xz(AZH^i1|7SFIgdkb^ z)n&y^TKDddwFlHAKjoiFO($ALo(f#j1>(u;6OC!?%8mFwpFniQUrac-^If0 z>0%`^$IGPPMh(w_wU@l}PUY7ZewPb~{Qwr(q&xj%s5>mhxYdHg`d0e?j^V2rf~mZ^ zDMz~}p1d>4Gj8ydtsU6fvR!?|H6|CmORKbLTLkg^$rU5z?OvYyP=-VZdH{jGNzFZr zf2_<#|33qW2G6CZH6C**f^4(ps?(8Th58BX#XYzq%Nu_2Xp-p(fd3AvT+Vg=r%uPI z`DofdTgXJ8R?mVjf(r!rDqqh-Dx622;Buc3bi;u`CV; zpc04bW&_bVT1mjMXuw(}{!8`nO=_>ufC}QjP~kVQWc_IYx2K+ss47C}iR`M*{~`ja z`y9;Zv@_IJ-fEH46&0j;vECKUtj5ic}az8rn~Oc#on~JVnR0I)iU)9 zCwfdgUPh~~c&S$=2{3?zMBSfQ^HJyQuU4kT{?a2VK@#athW!HASZ|ic={{o_louq_ z%sOr+|CII5PJ9OoL+_3)<4n@Xi7O;Rp3IP*Qb*QH`~^FsXo6gr5p_Di!v9Lf zN!?n$-ss|GQ)Z_MgVGg}lgV3z0Z=D%|Nvk5sdQf22`)LDEK^Yp1iQ^!m zjlrAjkLkbql|uGm$N=CF<8$X5QA8S)no1NYvDs>kernX1Gtjl44ITdA zxPo}FdKsKo;xkog4xi?C^t9>7yy*w5^0pscKSoWuaaBoJeiZw?p^F>lW8eo z;1qLDwxGo}s!BVpr(3U=MhF!F)~i+jDfv-_lob)1)lVa=CH4_amGUti$g2F7I|xl_ z+e7Wh_6eP%*SPG~OBT)lG#tqK1|}>2Hk|vb(ypFna!R4GNFpLFElls7qGc$WjmNFAX9cKR1gob`67bjgmoI~%8 zM<|3DSfL3u#yhUv->pqh*;Jx)?6jG(e10xNhBziyRgg_M(9`qRDbF)L@B(+pSuBS}unq!UT%|N)^jg4j}jH3o zJbAGj;XEBy(qHEBbii!WL7^tnV4nBvnnM|i$RsKdUFFS87smy-l{yJNmt>Yf!98ox z0z1)M|8z<#nn*gmS}~-^F_3WzR;pPKW$Ejd0?lH_puX;EuMpGZyqS16vzij13E;w$ zT-ql7o-o+Y21-gjGgvRi)V`u9@cNs6#_?VrNW!vml_XZ6^N?!W>qz@rmYVN+xw@L) z(1|YzbV8xIee#rT;fm*^al)(n)pTCmByfx<0tzQ^=fp63|F?jj0YS^isG|@rfRHV8 zIAgJM3hd}eAM8Cu4KnM5_Yn|HpM#HWE$-9>Gy0zPf0SAnI>Y;Szjo)@xt|(x6|!^2 KfN%pp!uUTw2GimI literal 0 HcmV?d00001 diff --git a/src/tools/test/tracejob/server_logs/20161003 b/src/tools/test/tracejob/server_logs/20161003 new file mode 100644 index 0000000000..5448ced5f2 --- /dev/null +++ b/src/tools/test/tracejob/server_logs/20161003 @@ -0,0 +1,9269 @@ +10/03/2016 00:00:03.976;02;PBS_Server.20704;Svr;Log;Log opened +10/03/2016 00:00:03.976;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:00:04.197;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:00:07.377;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:00:09.687;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:00:20.344;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:00:25.208;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:00:39.611;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:00:40.910;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:00:48.049;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:00:49.259;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:00:52.446;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:00:54.745;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:01:05.409;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:01:10.269;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:01:24.676;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:01:25.969;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:01:33.108;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:01:34.327;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:01:37.511;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:01:39.804;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:01:50.471;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:01:55.328;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:02:09.731;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:02:10.025;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:02:18.172;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:02:19.400;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:02:22.582;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:02:24.862;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:02:35.528;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:02:40.392;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:02:54.802;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:02:55.084;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:03:03.229;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:03:04.479;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:03:07.643;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:03:09.917;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:03:20.593;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:03:25.457;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:03:39.854;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:03:40.147;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:03:48.294;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:03:49.549;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:03:52.701;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:03:54.979;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:04:05.657;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:04:10.509;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:04:11.229;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:04:24.920;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:04:25.217;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:04:33.347;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:04:34.618;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:04:37.773;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:04:39.026;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:04:50.722;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:04:55.568;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:05:09.979;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:05:10.282;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:05:18.397;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:05:19.688;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:05:22.841;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:05:24.082;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:05:35.785;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:05:40.625;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:05:54.032;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:05:55.349;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:06:03.467;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:06:04.759;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:06:07.912;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:06:09.139;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:06:20.850;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:06:25.703;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:06:39.089;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:06:40.424;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:06:48.527;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:06:49.828;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:06:52.980;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:06:54.188;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:07:05.915;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:07:10.764;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:07:24.157;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:07:25.502;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:07:33.614;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:07:34.905;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:07:37.041;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:07:39.252;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:07:50.979;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:07:55.833;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:08:09.216;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:08:10.579;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:08:18.678;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:08:19.977;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:08:22.110;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:08:24.304;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:08:35.043;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:08:40.893;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:08:54.260;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:08:55.652;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:09:03.745;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:09:04.048;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:09:07.181;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:09:09.359;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:09:11.076;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:09:20.109;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:09:25.954;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:09:39.320;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:09:40.707;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:09:48.811;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:09:49.117;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:09:52.250;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:09:54.418;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:10:05.150;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:10:10.008;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:10:24.386;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:10:25.775;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:10:33.864;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:10:34.177;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:10:37.316;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:10:39.488;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:10:50.212;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:10:55.071;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:11:09.447;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:11:10.847;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:11:18.929;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:11:19.244;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:11:22.379;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:11:24.550;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:11:35.278;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:11:40.128;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:11:54.501;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:11:55.916;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:12:03.995;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:12:04.312;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:12:07.454;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:12:09.607;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:12:20.337;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:12:25.187;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:12:39.551;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:12:40.990;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:12:48.079;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:12:49.383;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:12:52.523;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:12:54.657;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:13:05.391;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:13:10.248;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:13:24.611;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:13:25.057;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:13:33.140;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:13:34.454;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:13:37.584;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:13:39.703;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:13:50.452;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:13:55.311;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:14:09.675;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:14:10.127;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:14:11.171;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:14:18.205;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:14:19.522;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:14:22.653;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:14:24.768;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:14:35.509;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:14:40.535;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:14:54.731;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:14:55.204;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:15:03.269;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:15:04.561;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:15:07.700;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:15:09.828;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:15:20.574;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:15:25.596;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:15:39.796;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:15:40.270;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:15:48.319;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:15:49.628;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:15:52.763;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:15:54.882;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:16:05.639;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:16:10.656;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:16:24.851;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:16:25.330;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:16:33.383;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:16:34.693;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:16:37.820;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:16:39.941;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:16:50.708;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:16:55.718;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:17:09.916;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:17:10.396;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:17:18.439;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:17:19.761;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:17:22.891;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:17:24.989;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:17:35.775;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:17:40.780;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:17:54.982;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:17:55.469;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:18:03.491;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:18:04.830;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:18:07.956;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:18:09.045;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:18:20.835;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:18:25.842;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:18:39.039;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:18:40.539;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:18:48.550;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:18:49.900;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:18:52.019;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:18:54.101;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:19:05.920;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:19:10.902;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:19:11.012;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:19:24.104;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:19:25.613;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:19:33.609;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:19:34.970;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:19:37.086;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:19:39.157;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:19:50.972;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:19:55.964;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:20:09.151;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:20:10.670;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:20:18.669;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:20:19.033;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:20:22.159;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:20:24.207;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:20:35.031;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:20:40.025;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:20:54.208;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:20:55.699;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:21:03.733;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:21:04.085;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:21:07.215;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:21:09.264;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:21:20.097;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:21:25.086;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:21:39.274;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:21:40.767;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:21:48.798;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:21:49.155;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:21:52.280;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:21:54.319;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:22:05.183;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:22:10.138;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:22:24.339;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:22:25.830;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:22:33.857;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:22:34.225;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:22:37.349;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:22:39.377;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:22:50.242;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:22:55.198;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:23:09.398;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:23:10.901;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:23:18.920;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:23:19.294;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:23:22.418;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:23:24.429;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:23:35.303;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:23:40.259;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:23:54.461;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:23:55.972;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:24:03.985;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:24:04.362;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:24:07.488;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:24:09.490;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:24:11.105;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:24:20.369;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:24:25.320;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:24:39.534;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:24:40.038;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:24:48.039;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:24:49.432;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:24:52.554;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:24:54.537;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:25:05.432;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:25:10.383;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:25:24.598;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:25:25.105;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:25:33.102;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:25:34.490;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:25:37.624;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:25:39.594;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:25:50.492;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:25:55.438;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:26:09.658;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:26:10.174;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:26:18.158;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:26:19.557;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:26:22.689;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:26:24.651;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:26:35.546;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:26:40.501;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:26:54.722;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:26:55.236;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:27:03.220;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:27:04.623;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:27:07.757;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:27:09.698;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:27:20.597;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:27:25.555;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:27:39.809;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:27:40.306;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:27:48.279;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:27:49.694;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:27:52.825;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:27:54.765;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:28:05.662;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:28:10.615;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:28:24.879;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:28:25.370;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:28:33.339;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:28:34.763;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:28:37.894;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:28:39.824;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:28:50.721;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:28:55.689;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:29:09.938;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:29:10.433;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:29:11.199;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:29:18.401;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:29:19.834;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:29:22.979;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:29:24.874;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:29:35.782;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:29:40.784;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:29:54.022;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:29:55.498;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:30:03.468;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:30:04.893;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:30:07.047;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:30:09.931;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:30:20.842;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:30:25.844;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:30:39.063;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:30:40.555;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:30:48.529;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:30:49.960;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:30:52.126;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:30:54.987;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:31:05.902;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:31:10.905;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:31:24.110;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:31:25.620;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:31:33.594;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:31:34.016;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:31:37.193;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:31:39.038;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:31:50.966;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:31:55.964;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:32:09.175;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:32:10.677;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:32:18.652;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:32:19.084;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:32:22.261;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:32:24.096;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:32:35.026;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:32:40.023;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:32:54.231;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:32:55.747;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:33:03.710;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:33:04.154;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:33:07.327;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:33:09.154;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:33:20.087;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:33:25.083;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:33:39.291;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:33:40.823;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:33:48.765;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:33:49.225;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:33:52.391;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:33:54.206;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:34:05.152;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:34:10.143;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:34:11.042;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:34:24.358;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:34:25.889;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:34:33.817;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:34:34.294;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:34:37.459;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:34:39.263;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:34:50.204;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:34:55.202;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:35:09.421;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:35:10.951;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:35:18.869;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:35:19.359;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:35:22.520;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:35:24.320;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:35:35.265;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:35:40.264;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:35:54.513;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:35:55.017;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:36:03.930;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:36:04.426;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:36:07.584;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:36:09.377;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:36:20.330;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:36:25.318;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:36:39.583;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:36:40.083;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:36:48.996;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:36:49.494;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:36:52.645;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:36:54.425;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:37:05.394;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:37:10.380;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:37:24.646;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:37:25.153;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:37:33.054;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:37:34.554;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:37:37.712;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:37:39.482;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:37:50.454;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:37:55.444;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:38:09.728;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:38:10.219;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:38:18.109;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:38:19.624;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:38:22.780;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:38:24.540;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:38:35.515;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:38:40.533;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:38:54.792;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:38:55.286;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:39:03.172;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:39:04.695;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:39:07.855;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:39:09.595;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:39:11.133;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:39:20.572;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:39:25.593;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:39:39.852;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:39:40.353;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:39:48.236;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:39:49.766;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:39:52.925;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:39:54.645;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:40:05.631;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:40:10.654;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:40:24.917;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:40:25.421;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:40:33.301;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:40:34.829;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:40:37.993;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:40:39.686;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:40:50.684;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:40:55.715;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:41:09.982;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:41:10.490;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:41:18.359;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:41:19.889;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:41:22.046;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:41:24.747;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:41:35.741;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:41:40.775;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:41:54.043;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:41:55.559;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:42:03.425;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:42:04.960;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:42:07.113;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:42:09.805;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:42:20.805;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:42:25.834;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:42:39.131;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:42:40.628;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:42:48.489;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:42:49.029;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:42:52.181;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:42:54.863;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:43:05.870;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:43:10.888;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:43:24.194;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:43:25.697;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:43:33.549;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:43:34.085;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:43:37.248;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:43:39.916;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:43:50.933;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:43:55.948;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:44:09.251;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:44:10.768;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:44:11.224;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:44:18.610;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:44:19.156;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:44:22.318;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:44:24.975;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:44:35.981;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:44:41.012;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:44:54.301;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:44:55.838;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:45:03.675;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:45:04.210;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:45:07.381;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:45:09.032;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:45:20.041;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:45:26.070;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:45:39.361;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:45:40.907;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:45:48.740;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:45:49.276;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:45:52.449;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:45:54.085;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:46:05.107;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:46:11.130;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:46:24.441;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:46:25.974;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:46:33.824;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:46:34.341;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:46:37.516;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:46:39.160;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:46:50.170;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:46:56.186;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:47:09.492;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:47:10.034;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:47:18.879;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:47:19.407;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:47:22.580;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:47:24.217;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:47:35.218;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:47:41.246;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:47:46.963;04;PBS_Server.28038;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:46.963;04;PBS_Server.28038;Svr;PBS_Server;attributes set: acl_host_enable = 0 +10/03/2016 00:47:46.983;04;PBS_Server.28039;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:46.983;04;PBS_Server.28039;Svr;PBS_Server;attributes set: acl_hosts = master23.banette.os,master23.banette.gent.vsc,master.banette.gent.vsc,master24.banette.gent.vsc +10/03/2016 00:47:46.002;04;PBS_Server.6728;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:46.002;04;PBS_Server.6728;Svr;PBS_Server;attributes set: checkpoint_dir = /export/home/checkpoints +10/03/2016 00:47:47.020;04;PBS_Server.6182;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.020;04;PBS_Server.6182;Svr;PBS_Server;attributes set: default_node = 1 +10/03/2016 00:47:47.039;04;PBS_Server.6210;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.039;04;PBS_Server.6210;Svr;PBS_Server;attributes set: default_queue = default +10/03/2016 00:47:47.048;04;PBS_Server.20704;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.049;04;PBS_Server.20704;Svr;PBS_Server;attributes set: disable_server_id_check = 1 +10/03/2016 00:47:47.067;04;PBS_Server.7306;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.067;04;PBS_Server.7306;Svr;PBS_Server;attributes set: down_on_error = 1 +10/03/2016 00:47:47.086;04;PBS_Server.20703;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.086;04;PBS_Server.20703;Svr;PBS_Server;attributes set: job_stat_rate = 90 +10/03/2016 00:47:47.094;04;PBS_Server.28038;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.094;04;PBS_Server.28038;Svr;PBS_Server;attributes set: legacy_vmem = 1 +10/03/2016 00:47:47.103;04;PBS_Server.28039;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.103;04;PBS_Server.28039;Svr;PBS_Server;attributes set: log_events = 511 +10/03/2016 00:47:47.122;04;PBS_Server.6728;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.122;04;PBS_Server.6728;Svr;PBS_Server;attributes set: log_file_max_size = 0 +10/03/2016 00:47:47.132;04;PBS_Server.6182;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.132;04;PBS_Server.6182;Svr;PBS_Server;attributes set: log_file_roll_depth = 10 +10/03/2016 00:47:47.150;04;PBS_Server.6210;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.150;04;PBS_Server.6210;Svr;PBS_Server;attributes set: log_level = 2 +10/03/2016 00:47:47.160;04;PBS_Server.20704;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.160;04;PBS_Server.20704;Svr;PBS_Server;attributes set: mail_domain = vscentrum.be +10/03/2016 00:47:47.179;04;PBS_Server.7306;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.179;04;PBS_Server.7306;Svr;PBS_Server;attributes set: mail_from = hpc-admin@lists.ugent.be +10/03/2016 00:47:47.187;04;PBS_Server.20703;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.187;04;PBS_Server.20703;Svr;PBS_Server;attributes set: managers = root@master.banette.gent.vsc,root@master.banette.os,root@master23.banette.gent.vsc,root@master23.banette.os,root@master24.banette.gent.vsc,root@master24.banette.os +10/03/2016 00:47:47.209;04;PBS_Server.28038;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.209;04;PBS_Server.28038;Svr;PBS_Server;attributes set: max_threads = 200 +10/03/2016 00:47:47.230;04;PBS_Server.28039;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.230;04;PBS_Server.28039;Svr;PBS_Server;attributes set: max_user_queuable = 10000 +10/03/2016 00:47:47.252;04;PBS_Server.6728;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.252;04;PBS_Server.6728;Svr;PBS_Server;attributes set: min_threads = 20 +10/03/2016 00:47:47.263;04;PBS_Server.6182;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.263;04;PBS_Server.6182;Svr;PBS_Server;attributes set: moab_array_compatible = 1 +10/03/2016 00:47:47.274;04;PBS_Server.6210;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.274;04;PBS_Server.6210;Svr;PBS_Server;attributes set: mom_job_sync = 1 +10/03/2016 00:47:47.286;04;PBS_Server.20704;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.286;04;PBS_Server.20704;Svr;PBS_Server;attributes set: no_mail_force = 1 +10/03/2016 00:47:47.307;04;PBS_Server.7306;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.307;04;PBS_Server.7306;Svr;PBS_Server;attributes set: node_check_rate = 600 +10/03/2016 00:47:47.328;04;PBS_Server.20703;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.328;04;PBS_Server.20703;Svr;PBS_Server;attributes set: node_pack = 1 +10/03/2016 00:47:47.350;04;PBS_Server.28038;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.350;04;PBS_Server.28038;Svr;PBS_Server;attributes set: node_ping_rate = 300 +10/03/2016 00:47:47.372;04;PBS_Server.28039;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.372;04;PBS_Server.28039;Svr;PBS_Server;attributes set: operators = root@master.banette.gent.vsc,root@master.banette.os,root@master23.banette.gent.vsc,root@master23.banette.os,root@master24.banette.gent.vsc,root@master24.banette.os +10/03/2016 00:47:47.393;04;PBS_Server.6728;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.393;04;PBS_Server.6728;Svr;PBS_Server;attributes set: poll_jobs = 1 +10/03/2016 00:47:47.415;04;PBS_Server.6182;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.415;04;PBS_Server.6182;Svr;PBS_Server;attributes set: query_other_jobs = 0 +10/03/2016 00:47:47.436;04;PBS_Server.6210;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.436;04;PBS_Server.6210;Svr;PBS_Server;attributes set: resources_default.nodect = 1 +10/03/2016 00:47:47.446;04;PBS_Server.20704;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.446;04;PBS_Server.20704;Svr;PBS_Server;attributes set: resources_default.nodes = 1 +10/03/2016 00:47:47.457;04;PBS_Server.7306;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.457;04;PBS_Server.7306;Svr;PBS_Server;attributes set: scheduler_iteration = 120 +10/03/2016 00:47:47.478;04;PBS_Server.20703;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.478;04;PBS_Server.20703;Svr;PBS_Server;attributes set: scheduling = 1 +10/03/2016 00:47:47.499;04;PBS_Server.28038;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.499;04;PBS_Server.28038;Svr;PBS_Server;attributes set: server_name = master23.banette.gent.vsc +10/03/2016 00:47:47.520;04;PBS_Server.28039;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.520;04;PBS_Server.28039;Svr;PBS_Server;attributes set: submit_hosts = master13,master15,master17,master19,master21,master23,master24,gligar01,gligar02,gligar03 +10/03/2016 00:47:47.533;04;PBS_Server.6728;Svr;PBS_Server;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.533;04;PBS_Server.6728;Svr;PBS_Server;attributes set: tcp_timeout = 120 +10/03/2016 00:47:47.545;04;PBS_Server.6182;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.545;04;PBS_Server.6182;Que;bshort;attributes set: Priority = 100 +10/03/2016 00:47:47.566;04;PBS_Server.6210;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.566;04;PBS_Server.6210;Que;bshort;attributes set: acl_group_enable = 1 +10/03/2016 00:47:47.578;04;PBS_Server.20704;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.578;04;PBS_Server.20704;Que;bshort;attributes set: acl_group_sloppy = 1 +10/03/2016 00:47:47.589;04;PBS_Server.7306;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.589;04;PBS_Server.7306;Que;bshort;attributes set: acl_groups = gadminforever,wheel +10/03/2016 00:47:47.600;04;PBS_Server.20703;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.600;04;PBS_Server.20703;Que;bshort;attributes set: queue_type = Execution +10/03/2016 00:47:47.610;04;PBS_Server.28038;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.610;04;PBS_Server.28038;Que;bshort;attributes set: resources_default.nice = 0 +10/03/2016 00:47:47.631;04;PBS_Server.28039;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.631;04;PBS_Server.28039;Que;bshort;attributes set: resources_default.walltime = 01:00:00 +10/03/2016 00:47:47.641;04;PBS_Server.6728;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.641;04;PBS_Server.6728;Que;bshort;attributes set: resources_max.walltime = 11:59:59 +10/03/2016 00:47:47.662;04;PBS_Server.6182;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.662;04;PBS_Server.6182;Que;bshort;attributes set: enabled = 1 +10/03/2016 00:47:47.672;04;PBS_Server.6210;Que;bshort;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.673;04;PBS_Server.6210;Que;bshort;attributes set: started = 1 +10/03/2016 00:47:47.683;04;PBS_Server.20704;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.683;04;PBS_Server.20704;Que;debug;attributes set: Priority = 60 +10/03/2016 00:47:47.693;04;PBS_Server.7306;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.693;04;PBS_Server.7306;Que;debug;attributes set: acl_group_enable = 1 +10/03/2016 00:47:47.703;04;PBS_Server.20703;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.704;04;PBS_Server.20703;Que;debug;attributes set: acl_group_sloppy = 1 +10/03/2016 00:47:47.714;04;PBS_Server.28038;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.714;04;PBS_Server.28038;Que;debug;attributes set: acl_groups = gadminforever,wheel +10/03/2016 00:47:47.724;04;PBS_Server.28039;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.725;04;PBS_Server.28039;Que;debug;attributes set: queue_type = Execution +10/03/2016 00:47:47.735;04;PBS_Server.6728;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.735;04;PBS_Server.6728;Que;debug;attributes set: resources_default.nice = 0 +10/03/2016 00:47:47.755;04;PBS_Server.6182;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.756;04;PBS_Server.6182;Que;debug;attributes set: resources_default.walltime = 00:15:00 +10/03/2016 00:47:47.776;04;PBS_Server.6210;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.776;04;PBS_Server.6210;Que;debug;attributes set: resources_max.walltime = 00:59:59 +10/03/2016 00:47:47.786;04;PBS_Server.20704;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.787;04;PBS_Server.20704;Que;debug;attributes set: enabled = 1 +10/03/2016 00:47:47.797;04;PBS_Server.7306;Que;debug;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.797;04;PBS_Server.7306;Que;debug;attributes set: started = 1 +10/03/2016 00:47:47.807;04;PBS_Server.20703;Que;default;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.807;04;PBS_Server.20703;Que;default;attributes set: queue_type = Route +10/03/2016 00:47:47.818;04;PBS_Server.28038;Que;default;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.818;04;PBS_Server.28038;Que;default;attributes set: route_destinations = short,long +10/03/2016 00:47:47.829;04;PBS_Server.28039;Que;default;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.829;04;PBS_Server.28039;Que;default;attributes set: enabled = 1 +10/03/2016 00:47:47.839;04;PBS_Server.6728;Que;default;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.839;04;PBS_Server.6728;Que;default;attributes set: started = 1 +10/03/2016 00:47:47.860;04;PBS_Server.6182;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.860;04;PBS_Server.6182;Que;long;attributes set: Priority = 60 +10/03/2016 00:47:47.870;04;PBS_Server.6210;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.871;04;PBS_Server.6210;Que;long;attributes set: acl_group_enable = 1 +10/03/2016 00:47:47.881;04;PBS_Server.20704;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.881;04;PBS_Server.20704;Que;long;attributes set: acl_group_sloppy = 1 +10/03/2016 00:47:47.891;04;PBS_Server.7306;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.892;04;PBS_Server.7306;Que;long;attributes set: acl_groups = gadminforever,wheel +10/03/2016 00:47:47.902;04;PBS_Server.20703;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.902;04;PBS_Server.20703;Que;long;attributes set: queue_type = Execution +10/03/2016 00:47:47.912;04;PBS_Server.28038;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.913;04;PBS_Server.28038;Que;long;attributes set: resources_default.nice = 0 +10/03/2016 00:47:47.923;04;PBS_Server.28039;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.923;04;PBS_Server.28039;Que;long;attributes set: resources_default.walltime = 48:00:00 +10/03/2016 00:47:47.933;04;PBS_Server.6728;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.933;04;PBS_Server.6728;Que;long;attributes set: resources_max.walltime = 72:00:00 +10/03/2016 00:47:47.944;04;PBS_Server.6182;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.944;04;PBS_Server.6182;Que;long;attributes set: enabled = 1 +10/03/2016 00:47:47.964;04;PBS_Server.6210;Que;long;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.964;04;PBS_Server.6210;Que;long;attributes set: started = 1 +10/03/2016 00:47:47.974;04;PBS_Server.20704;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.974;04;PBS_Server.20704;Que;q1h;attributes set: Priority = 60 +10/03/2016 00:47:47.985;04;PBS_Server.7306;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.985;04;PBS_Server.7306;Que;q1h;attributes set: acl_group_enable = 1 +10/03/2016 00:47:47.995;04;PBS_Server.20703;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.996;04;PBS_Server.20703;Que;q1h;attributes set: acl_group_sloppy = 1 +10/03/2016 00:47:47.006;04;PBS_Server.28038;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:47.006;04;PBS_Server.28038;Que;q1h;attributes set: acl_groups = gadminforever,wheel +10/03/2016 00:47:48.027;04;PBS_Server.28039;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.027;04;PBS_Server.28039;Que;q1h;attributes set: queue_type = Execution +10/03/2016 00:47:48.047;04;PBS_Server.6728;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.047;04;PBS_Server.6728;Que;q1h;attributes set: resources_default.nice = 0 +10/03/2016 00:47:48.058;04;PBS_Server.6182;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.058;04;PBS_Server.6182;Que;q1h;attributes set: resources_default.walltime = 01:00:00 +10/03/2016 00:47:48.078;04;PBS_Server.6210;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.079;04;PBS_Server.6210;Que;q1h;attributes set: resources_max.walltime = 01:00:00 +10/03/2016 00:47:48.090;04;PBS_Server.20704;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.090;04;PBS_Server.20704;Que;q1h;attributes set: resources_min.walltime = 00:00:00 +10/03/2016 00:47:48.111;04;PBS_Server.6728;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.111;04;PBS_Server.6728;Que;q1h;attributes set: enabled = 1 +10/03/2016 00:47:48.121;04;PBS_Server.20703;Que;q1h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.122;04;PBS_Server.20703;Que;q1h;attributes set: started = 1 +10/03/2016 00:47:48.143;04;PBS_Server.28038;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.143;04;PBS_Server.28038;Que;q24h;attributes set: Priority = 60 +10/03/2016 00:47:48.153;04;PBS_Server.28039;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.153;04;PBS_Server.28039;Que;q24h;attributes set: acl_group_enable = 1 +10/03/2016 00:47:48.174;04;PBS_Server.7306;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.174;04;PBS_Server.7306;Que;q24h;attributes set: acl_group_sloppy = 1 +10/03/2016 00:47:48.184;04;PBS_Server.6182;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.185;04;PBS_Server.6182;Que;q24h;attributes set: acl_groups = gadminforever,wheel +10/03/2016 00:47:48.206;04;PBS_Server.6210;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.206;04;PBS_Server.6210;Que;q24h;attributes set: queue_type = Execution +10/03/2016 00:47:48.227;04;PBS_Server.20704;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.227;04;PBS_Server.20704;Que;q24h;attributes set: resources_default.nice = 0 +10/03/2016 00:47:48.248;04;PBS_Server.6728;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.248;04;PBS_Server.6728;Que;q24h;attributes set: resources_default.walltime = 12:00:00 +10/03/2016 00:47:48.269;04;PBS_Server.20703;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.269;04;PBS_Server.20703;Que;q24h;attributes set: resources_max.walltime = 24:00:00 +10/03/2016 00:47:48.290;04;PBS_Server.28038;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.291;04;PBS_Server.28038;Que;q24h;attributes set: resources_min.walltime = 01:00:00 +10/03/2016 00:47:48.311;04;PBS_Server.28039;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.312;04;PBS_Server.28039;Que;q24h;attributes set: enabled = 1 +10/03/2016 00:47:48.332;04;PBS_Server.7306;Que;q24h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.332;04;PBS_Server.7306;Que;q24h;attributes set: started = 1 +10/03/2016 00:47:48.343;04;PBS_Server.6182;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.343;04;PBS_Server.6182;Que;q72h;attributes set: Priority = 60 +10/03/2016 00:47:48.364;04;PBS_Server.6210;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.365;04;PBS_Server.6210;Que;q72h;attributes set: acl_group_enable = 1 +10/03/2016 00:47:48.375;04;PBS_Server.20704;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.375;04;PBS_Server.20704;Que;q72h;attributes set: acl_group_sloppy = 1 +10/03/2016 00:47:48.386;04;PBS_Server.6728;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.386;04;PBS_Server.6728;Que;q72h;attributes set: acl_groups = gadminforever,wheel +10/03/2016 00:47:48.397;04;PBS_Server.20703;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.397;04;PBS_Server.20703;Que;q72h;attributes set: queue_type = Execution +10/03/2016 00:47:48.418;04;PBS_Server.28038;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.418;04;PBS_Server.28038;Que;q72h;attributes set: resources_default.nice = 0 +10/03/2016 00:47:48.439;04;PBS_Server.28039;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.439;04;PBS_Server.28039;Que;q72h;attributes set: resources_default.walltime = 48:00:00 +10/03/2016 00:47:48.450;04;PBS_Server.7306;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.452;04;PBS_Server.7306;Que;q72h;attributes set: resources_max.walltime = 72:00:00 +10/03/2016 00:47:48.463;04;PBS_Server.6182;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.463;04;PBS_Server.6182;Que;q72h;attributes set: resources_min.walltime = 24:00:00 +10/03/2016 00:47:48.484;04;PBS_Server.6210;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.484;04;PBS_Server.6210;Que;q72h;attributes set: enabled = 1 +10/03/2016 00:47:48.505;04;PBS_Server.20704;Que;q72h;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.505;04;PBS_Server.20704;Que;q72h;attributes set: started = 1 +10/03/2016 00:47:48.526;04;PBS_Server.6728;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.526;04;PBS_Server.6728;Que;short;attributes set: Priority = 100 +10/03/2016 00:47:48.536;04;PBS_Server.20703;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.537;04;PBS_Server.20703;Que;short;attributes set: acl_group_enable = 1 +10/03/2016 00:47:48.558;04;PBS_Server.28038;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.558;04;PBS_Server.28038;Que;short;attributes set: acl_group_sloppy = 1 +10/03/2016 00:47:48.579;04;PBS_Server.28039;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.579;04;PBS_Server.28039;Que;short;attributes set: acl_groups = gadminforever,wheel +10/03/2016 00:47:48.601;04;PBS_Server.7306;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.602;04;PBS_Server.7306;Que;short;attributes set: queue_type = Execution +10/03/2016 00:47:48.624;04;PBS_Server.6182;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.624;04;PBS_Server.6182;Que;short;attributes set: resources_default.nice = 0 +10/03/2016 00:47:48.647;04;PBS_Server.6210;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.647;04;PBS_Server.6210;Que;short;attributes set: resources_default.walltime = 01:00:00 +10/03/2016 00:47:48.669;04;PBS_Server.20704;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.669;04;PBS_Server.20704;Que;short;attributes set: resources_max.walltime = 11:59:59 +10/03/2016 00:47:48.692;04;PBS_Server.6728;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.692;04;PBS_Server.6728;Que;short;attributes set: enabled = 1 +10/03/2016 00:47:48.705;04;PBS_Server.20703;Que;short;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.705;04;PBS_Server.20703;Que;short;attributes set: started = 1 +10/03/2016 00:47:48.717;04;PBS_Server.28038;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.718;04;PBS_Server.28038;Que;special;attributes set: Priority = 60 +10/03/2016 00:47:48.740;04;PBS_Server.28039;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.740;04;PBS_Server.28039;Que;special;attributes set: acl_group_enable = 1 +10/03/2016 00:47:48.762;04;PBS_Server.7306;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.763;04;PBS_Server.7306;Que;special;attributes set: acl_group_sloppy = 1 +10/03/2016 00:47:48.785;04;PBS_Server.6182;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.815;04;PBS_Server.6182;Que;special;attributes set: acl_groups = gadminforever,wheel +10/03/2016 00:47:48.827;04;PBS_Server.6210;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.827;04;PBS_Server.6210;Que;special;attributes set: queue_type = Execution +10/03/2016 00:47:48.848;04;PBS_Server.20704;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.848;04;PBS_Server.20704;Que;special;attributes set: resources_default.nice = 0 +10/03/2016 00:47:48.869;04;PBS_Server.6728;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.869;04;PBS_Server.6728;Que;special;attributes set: resources_default.walltime = 48:00:00 +10/03/2016 00:47:48.881;04;PBS_Server.20703;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.881;04;PBS_Server.20703;Que;special;attributes set: resources_max.walltime = 144:00:00 +10/03/2016 00:47:48.902;04;PBS_Server.28038;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.902;04;PBS_Server.28038;Que;special;attributes set: enabled = 1 +10/03/2016 00:47:48.924;04;PBS_Server.28039;Que;special;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.924;04;PBS_Server.28039;Que;special;attributes set: started = 1 +10/03/2016 00:47:48.945;04;PBS_Server.7306;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.945;04;PBS_Server.7306;Que;workshop;attributes set: Priority = 100 +10/03/2016 00:47:48.967;04;PBS_Server.6182;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.967;04;PBS_Server.6182;Que;workshop;attributes set: acl_group_enable = 1 +10/03/2016 00:47:48.978;04;PBS_Server.6210;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.978;04;PBS_Server.6210;Que;workshop;attributes set: acl_group_sloppy = 1 +10/03/2016 00:47:48.000;04;PBS_Server.20704;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.000;04;PBS_Server.20704;Que;workshop;attributes set: acl_groups = gworkshop +10/03/2016 00:47:48.013;04;PBS_Server.6728;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:48.014;04;PBS_Server.6728;Que;workshop;attributes set: queue_type = Execution +10/03/2016 00:47:49.025;04;PBS_Server.20703;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.026;04;PBS_Server.20703;Que;workshop;attributes set: resources_default.nice = 0 +10/03/2016 00:47:49.048;04;PBS_Server.28038;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.048;04;PBS_Server.28038;Que;workshop;attributes set: resources_default.walltime = 01:00:00 +10/03/2016 00:47:49.070;04;PBS_Server.28039;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.070;04;PBS_Server.28039;Que;workshop;attributes set: resources_max.walltime = 72:00:00 +10/03/2016 00:47:49.083;04;PBS_Server.7306;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.083;04;PBS_Server.7306;Que;workshop;attributes set: enabled = 1 +10/03/2016 00:47:49.095;04;PBS_Server.6182;Que;workshop;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.096;04;PBS_Server.6182;Que;workshop;attributes set: started = 1 +10/03/2016 00:47:49.131;04;PBS_Server.20704;node;node2801.banette.gent.vsc;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.131;04;PBS_Server.20704;node;node2801.banette.gent.vsc;attributes set: np = 2 +10/03/2016 00:47:49.154;04;PBS_Server.6728;node;node2802.banette.gent.vsc;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.154;04;PBS_Server.6728;node;node2802.banette.gent.vsc;attributes set: np = 2 +10/03/2016 00:47:49.166;04;PBS_Server.20703;node;node2803.banette.gent.vsc;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.166;04;PBS_Server.20703;node;node2803.banette.gent.vsc;attributes set: np = 2 +10/03/2016 00:47:49.188;04;PBS_Server.28038;node;node2804.banette.gent.vsc;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.188;04;PBS_Server.28038;node;node2804.banette.gent.vsc;attributes set: np = 2 +10/03/2016 00:47:49.208;04;PBS_Server.28039;node;node2805.banette.gent.vsc;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.208;04;PBS_Server.28039;node;node2805.banette.gent.vsc;attributes set: np = 2 +10/03/2016 00:47:49.219;04;PBS_Server.7306;node;node2806.banette.gent.vsc;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.219;04;PBS_Server.7306;node;node2806.banette.gent.vsc;attributes set: np = 2 +10/03/2016 00:47:49.229;04;PBS_Server.6182;node;node2807.banette.gent.vsc;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.229;04;PBS_Server.6182;node;node2807.banette.gent.vsc;attributes set: np = 2 +10/03/2016 00:47:49.240;04;PBS_Server.6210;node;node2808.banette.gent.vsc;attributes set: at request of root@master23.banette.gent.vsc +10/03/2016 00:47:49.240;04;PBS_Server.6210;node;node2808.banette.gent.vsc;attributes set: np = 2 +10/03/2016 00:47:54.550;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:47:55.103;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:48:03.940;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:48:04.471;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:48:07.645;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:48:09.278;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:48:20.276;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:48:26.306;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:48:39.610;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:48:40.173;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:48:48.002;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:48:49.542;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:48:52.711;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:48:54.337;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:49:05.340;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:49:11.066;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:49:11.366;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:49:24.671;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:49:25.243;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:49:33.076;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:49:34.613;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:49:37.780;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:49:39.398;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:49:50.401;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:49:56.425;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:50:09.727;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:50:10.303;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:50:18.138;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:50:19.675;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:50:22.847;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:50:24.446;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:50:35.465;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:50:41.484;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:50:54.784;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:50:55.372;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:51:03.193;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:51:04.743;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:51:07.908;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:51:09.498;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:51:20.520;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:51:26.544;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:51:39.843;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:51:40.434;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:51:48.257;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:51:49.812;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:51:52.990;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:51:54.557;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:52:05.584;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:52:11.601;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:52:24.908;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:52:25.504;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:52:33.316;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:52:34.882;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:52:37.060;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:52:39.606;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:52:50.648;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:52:56.662;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:53:09.967;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:53:10.576;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:53:18.369;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:53:19.952;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:53:22.138;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:53:24.652;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:53:35.725;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:53:41.723;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:53:54.029;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:53:55.646;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:54:03.433;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:54:04.022;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:54:07.206;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:54:09.710;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:54:11.158;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:54:20.784;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:54:26.792;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:54:39.094;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:54:40.756;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:54:48.496;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:54:49.091;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:54:52.275;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:54:54.767;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:55:05.846;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:55:11.869;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:55:24.153;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:55:25.825;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:55:33.550;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:55:34.151;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:55:37.342;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:55:39.826;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:55:50.902;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:55:56.934;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:56:09.196;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:56:10.904;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:56:18.607;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:56:19.218;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:56:22.398;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:56:24.891;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:56:35.965;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:56:41.990;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:56:54.250;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:56:55.966;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:57:03.673;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:57:04.282;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:57:07.472;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:57:09.960;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:57:20.028;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:57:26.049;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:57:39.314;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:57:40.026;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:57:48.729;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:57:49.353;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:57:52.544;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:57:54.021;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:58:05.112;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:58:11.109;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:58:24.371;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:58:25.084;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:58:33.794;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:58:34.423;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:58:37.613;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:58:39.080;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:58:50.175;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:58:56.168;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:59:09.435;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:59:10.188;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 00:59:11.253;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 00:59:18.852;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 00:59:19.499;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 00:59:22.681;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 00:59:24.147;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 00:59:35.222;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 00:59:41.229;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 00:59:54.492;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 00:59:55.254;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:00:03.904;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:00:04.564;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:00:07.752;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:00:09.221;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:00:20.277;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:00:26.282;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:00:39.550;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:00:40.324;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:00:48.966;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:00:49.630;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:00:52.819;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:00:54.277;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:01:05.339;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:01:11.357;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:01:24.615;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:01:25.385;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:01:33.027;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:01:34.695;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:01:37.881;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:01:39.326;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:01:50.395;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:01:56.414;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:02:09.680;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:02:10.449;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:02:18.090;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:02:19.763;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:02:22.950;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:02:24.390;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:02:35.449;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:02:41.478;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:02:54.742;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:02:55.536;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:03:03.149;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:03:04.822;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:03:07.015;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:03:09.447;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:03:20.502;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:03:26.534;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:03:39.802;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:03:40.611;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:03:48.219;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:03:49.879;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:03:52.081;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:03:54.506;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:04:05.562;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:04:11.094;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:04:11.594;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:04:24.861;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:04:25.682;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:04:33.280;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:04:34.944;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:04:37.143;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:04:39.584;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:04:50.621;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:04:56.655;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:05:09.926;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:05:10.747;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:05:18.337;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:05:19.006;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:05:22.184;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:05:24.635;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:05:35.699;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:05:41.715;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:05:54.979;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:05:55.816;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:06:03.399;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:06:04.074;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:06:07.246;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:06:09.689;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:06:20.765;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:06:26.778;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:06:39.040;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:06:40.884;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:06:48.460;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:06:49.135;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:06:52.310;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:06:54.756;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:07:05.834;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:07:11.834;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:07:24.102;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:07:25.956;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:07:33.518;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:07:34.205;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:07:37.372;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:07:39.796;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:07:50.898;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:07:56.894;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:08:09.156;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:08:10.023;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:08:18.580;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:08:19.273;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:08:22.440;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:08:24.864;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:08:35.967;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:08:41.962;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:08:54.219;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:08:55.111;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:09:03.642;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:09:04.347;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:09:07.510;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:09:09.915;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:09:11.184;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:09:20.032;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:09:26.030;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:09:39.283;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:09:40.175;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:09:48.705;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:09:49.419;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:09:52.581;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:09:54.966;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:10:05.095;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:10:11.096;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:10:24.350;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:10:25.246;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:10:33.768;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:10:34.489;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:10:37.641;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:10:39.022;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:10:50.159;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:10:56.159;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:11:09.411;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:11:10.361;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:11:18.832;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:11:19.549;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:11:22.711;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:11:24.078;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:11:35.221;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:11:41.220;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:11:54.469;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:11:55.436;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:12:03.887;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:12:04.616;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:12:07.777;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:12:09.129;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:12:20.265;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:12:26.280;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:12:39.531;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:12:40.507;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:12:48.938;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:12:49.690;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:12:52.872;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:12:54.185;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:13:05.310;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:13:11.335;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:13:24.591;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:13:25.577;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:13:33.986;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:13:34.761;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:13:37.928;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:13:39.235;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:13:50.354;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:13:56.400;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:14:09.656;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:14:10.649;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:14:11.025;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:14:18.050;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:14:19.824;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:14:22.997;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:14:24.286;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:14:35.401;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:14:41.462;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:14:54.721;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:14:55.764;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:15:03.108;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:15:04.890;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:15:07.066;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:15:09.346;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:15:20.454;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:15:26.528;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:15:39.783;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:15:40.825;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:15:48.173;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:15:49.959;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:15:52.138;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:15:54.402;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:16:05.515;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:16:11.590;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:16:24.843;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:16:25.891;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:16:33.237;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:16:35.018;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:16:37.207;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:16:39.456;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:16:50.572;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:16:56.650;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:17:09.894;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:17:10.955;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:17:18.302;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:17:19.084;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:17:22.296;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:17:24.510;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:17:35.634;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:17:41.711;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:17:54.956;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:17:56.016;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:18:03.358;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:18:04.154;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:18:07.322;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:18:09.567;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:18:20.681;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:18:26.771;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:18:39.012;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:18:41.086;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:18:48.422;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:18:49.225;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:18:52.379;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:18:54.622;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:19:05.742;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:19:11.117;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:19:11.837;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:19:25.076;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:19:26.161;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:19:33.483;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:19:34.292;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:19:37.448;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:19:39.677;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:19:50.804;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:19:56.901;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:20:10.141;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:20:11.226;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:20:18.539;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:20:19.357;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:20:22.516;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:20:24.733;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:20:35.909;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:20:41.962;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:20:55.205;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:20:56.293;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:21:03.602;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:21:04.425;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:21:07.579;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:21:09.790;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:21:20.973;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:21:27.016;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:21:40.261;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:21:41.354;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:21:48.655;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:21:49.495;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:21:52.643;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:21:54.836;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:22:05.039;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:22:12.083;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:22:25.326;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:22:26.423;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:22:33.715;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:22:34.561;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:22:37.710;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:22:39.892;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:22:50.096;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:22:57.146;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:23:10.391;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:23:11.486;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:23:18.776;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:23:19.635;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:23:22.780;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:23:24.945;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:23:35.158;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:23:42.207;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:23:55.453;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:23:56.556;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:24:03.842;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:24:04.717;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:24:07.851;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:24:09.996;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:24:11.213;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:24:20.243;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:24:27.267;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:24:40.510;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:24:41.627;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:24:48.896;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:24:49.789;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:24:52.920;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:24:54.046;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:25:05.308;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:25:12.327;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:25:25.574;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:25:26.695;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:25:33.954;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:25:34.853;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:25:37.989;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:25:39.102;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:25:50.376;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:25:57.384;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:26:10.635;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:26:11.765;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:26:18.017;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:26:19.922;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:26:22.047;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:26:24.159;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:26:35.450;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:26:42.439;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:26:55.694;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:26:56.826;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:27:03.073;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:27:04.991;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:27:07.109;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:27:09.206;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:27:20.505;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:27:27.506;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:27:40.750;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:27:41.899;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:27:48.131;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:27:49.080;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:27:52.181;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:27:54.262;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:28:05.561;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:28:12.566;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:28:25.812;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:28:26.964;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:28:33.189;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:28:34.177;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:28:37.250;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:28:39.318;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:28:50.621;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:28:57.628;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:29:10.869;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:29:11.025;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:29:11.056;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:29:18.250;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:29:19.248;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:29:22.320;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:29:24.377;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:29:35.711;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:29:42.705;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:29:55.933;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:29:56.091;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:30:03.314;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:30:04.318;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:30:07.419;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:30:09.429;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:30:20.771;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:30:27.765;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:30:40.992;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:30:41.155;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:30:48.369;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:30:49.401;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:30:52.458;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:30:54.475;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:31:05.835;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:31:12.821;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:31:25.053;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:31:26.219;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:31:33.433;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:31:34.466;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:31:37.517;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:31:39.528;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:31:50.885;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:31:57.883;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:32:10.118;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:32:11.286;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:32:18.498;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:32:19.532;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:32:22.580;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:32:24.576;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:32:35.972;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:32:42.946;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:32:55.183;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:32:56.355;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:33:03.561;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:33:04.603;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:33:07.650;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:33:09.634;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:33:20.038;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:33:28.007;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:33:40.233;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:33:41.421;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:33:48.619;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:33:49.674;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:33:52.718;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:33:54.685;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:34:05.102;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:34:11.150;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:34:12.068;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:34:25.289;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:34:26.491;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:34:33.679;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:34:34.744;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:34:37.798;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:34:39.741;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:34:50.160;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:34:57.124;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:35:10.359;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:35:11.556;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:35:18.739;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:35:19.806;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:35:22.877;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:35:24.798;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:35:35.220;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:35:42.201;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:35:55.423;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:35:56.622;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:36:03.804;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:36:04.874;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:36:07.944;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:36:09.854;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:36:20.282;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:36:27.261;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:36:40.484;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:36:41.685;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:36:48.858;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:36:49.945;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:36:53.010;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:36:54.906;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:37:05.341;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:37:12.323;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:37:25.540;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:37:26.748;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:37:33.919;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:37:34.016;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:37:37.078;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:37:39.956;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:37:50.405;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:37:57.385;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:38:10.598;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:38:11.817;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:38:18.983;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:38:19.086;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:38:22.146;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:38:24.016;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:38:35.466;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:38:42.440;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:38:55.664;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:38:56.888;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:39:03.046;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:39:04.154;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:39:07.216;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:39:09.070;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:39:11.244;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:39:20.514;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:39:27.501;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:39:40.724;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:39:41.963;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:39:48.109;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:39:49.229;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:39:52.284;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:39:54.126;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:40:05.560;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:40:12.560;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:40:25.789;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:40:26.023;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:40:33.171;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:40:34.300;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:40:37.366;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:40:39.180;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:40:50.623;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:40:57.615;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:41:10.847;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:41:11.093;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:41:18.235;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:41:19.370;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:41:22.442;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:41:24.237;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:41:35.689;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:41:42.675;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:41:55.910;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:41:56.157;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:42:03.303;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:42:04.438;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:42:07.517;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:42:09.303;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:42:20.752;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:42:27.736;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:42:40.973;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:42:41.228;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:42:48.358;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:42:49.504;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:42:52.587;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:42:54.355;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:43:05.816;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:43:12.797;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:43:25.032;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:43:26.299;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:43:33.418;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:43:34.576;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:43:37.657;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:43:39.403;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:43:50.879;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:43:57.854;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:44:10.093;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:44:11.093;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:44:11.366;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:44:18.478;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:44:19.649;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:44:22.728;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:44:24.456;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:44:35.931;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:44:42.915;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:44:55.141;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:44:56.422;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:45:03.538;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:45:04.721;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:45:07.803;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:45:09.506;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:45:20.975;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:45:27.975;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:45:40.206;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:45:41.486;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:45:48.600;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:45:49.790;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:45:52.864;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:45:54.562;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:46:05.039;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:46:12.036;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:46:25.259;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:46:26.549;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:46:33.664;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:46:34.857;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:46:37.926;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:46:39.617;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:46:50.103;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:46:57.089;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:47:10.324;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:47:11.618;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:47:18.730;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:47:19.927;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:47:22.996;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:47:24.680;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:47:35.166;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:47:42.149;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:47:55.388;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:47:56.690;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:48:03.790;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:48:04.999;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:48:07.058;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:48:09.736;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:48:20.221;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:48:27.213;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:48:40.450;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:48:41.768;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:48:48.859;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:48:49.063;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:48:52.123;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:48:54.793;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:49:05.282;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:49:11.192;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:49:12.274;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:49:25.510;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:49:26.837;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:49:33.916;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:49:34.124;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:49:37.197;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:49:39.849;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:49:50.341;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:49:57.335;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:50:10.575;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:50:11.908;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:50:18.968;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:50:19.185;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:50:22.266;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:50:24.905;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:50:35.405;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:50:42.395;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:50:55.640;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:50:57.007;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:51:03.028;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:51:04.251;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:51:07.323;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:51:09.966;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:51:20.465;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:51:27.455;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:51:40.732;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:51:41.071;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:51:48.084;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:51:49.319;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:51:52.390;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:51:54.014;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:52:05.520;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:52:12.516;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:52:25.796;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:52:26.139;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:52:33.134;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:52:34.395;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:52:37.452;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:52:39.062;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:52:50.583;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:52:57.569;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:53:10.859;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:53:11.209;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:53:18.197;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:53:19.465;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:53:22.521;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:53:24.127;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:53:35.634;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:53:42.629;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:53:55.906;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:53:56.278;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:54:03.262;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:54:04.534;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:54:07.581;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:54:09.179;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:54:11.041;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:54:20.693;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:54:27.689;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:54:41.007;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:54:41.346;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:54:48.325;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:54:49.595;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:54:52.652;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:54:54.235;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:55:05.751;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:55:12.749;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:55:25.071;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:55:26.432;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:55:33.380;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:55:34.699;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:55:37.699;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:55:39.303;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:55:50.816;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:55:57.804;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:56:10.133;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:56:11.503;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:56:18.447;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:56:19.766;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:56:22.759;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:56:24.360;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:56:35.881;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:56:42.866;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:56:55.194;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:56:56.573;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:57:03.509;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:57:04.833;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:57:07.828;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:57:09.419;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:57:20.947;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:57:27.926;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:57:40.250;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:57:41.642;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:57:48.564;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:57:49.904;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:57:52.893;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:57:54.469;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:58:06.011;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:58:12.985;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:58:25.302;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:58:26.744;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:58:33.619;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:58:34.974;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:58:37.961;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:58:39.528;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:58:51.090;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:58:57.043;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:59:10.365;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:59:11.134;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 01:59:11.814;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 01:59:18.683;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 01:59:19.044;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 01:59:22.030;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 01:59:24.588;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 01:59:36.142;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 01:59:42.105;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 01:59:55.429;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 01:59:56.872;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:00:03.746;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:00:04.111;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:00:07.103;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:00:09.644;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:00:21.205;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:00:27.163;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:00:40.481;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:00:41.937;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:00:48.810;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:00:49.205;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:00:52.174;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:00:54.695;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:01:06.264;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:01:12.224;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:01:25.539;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:01:26.009;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:01:33.868;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:01:34.272;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:01:37.239;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:01:39.746;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:01:51.329;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:01:57.285;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:02:10.602;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:02:11.078;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:02:18.932;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:02:19.387;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:02:22.310;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:02:24.802;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:02:36.390;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:02:42.359;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:02:55.661;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:02:56.147;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:03:03.992;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:03:04.458;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:03:07.376;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:03:09.861;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:03:21.448;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:03:27.421;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:03:40.720;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:03:41.234;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:03:48.057;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:03:49.529;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:03:52.448;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:03:54.917;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:04:06.521;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:04:11.225;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:04:12.485;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:04:25.786;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:04:26.304;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:04:33.115;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:04:34.629;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:04:37.514;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:04:39.962;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:04:51.582;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:04:57.547;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:05:10.842;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:05:11.369;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:05:18.176;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:05:19.696;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:05:22.583;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:05:24.017;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:05:36.647;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:05:42.606;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:05:55.907;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:05:56.436;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:06:03.240;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:06:04.765;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:06:07.646;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:06:09.070;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:06:21.728;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:06:27.660;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:06:40.971;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:06:41.495;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:06:48.304;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:06:49.875;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:06:52.715;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:06:54.127;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:07:06.795;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:07:12.722;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:07:25.027;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:07:26.562;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:07:33.359;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:07:34.946;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:07:37.810;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:07:39.177;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:07:51.877;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:07:57.783;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:08:10.086;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:08:11.630;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:08:18.423;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:08:20.015;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:08:22.880;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:08:24.243;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:08:36.945;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:08:42.846;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:08:55.149;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:08:56.702;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:09:03.487;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:09:04.107;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:09:07.954;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:09:09.299;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:09:11.067;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:09:21.010;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:09:27.907;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:09:40.215;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:09:41.774;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:09:48.551;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:09:49.180;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:09:53.018;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:09:54.358;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:10:06.085;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:10:12.972;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:10:25.280;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:10:26.845;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:10:33.608;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:10:34.251;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:10:37.086;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:10:39.416;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:10:51.166;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:10:57.066;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:11:10.327;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:11:11.911;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:11:18.669;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:11:19.323;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:11:22.147;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:11:24.469;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:11:36.234;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:11:42.126;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:11:55.382;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:11:56.976;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:12:03.734;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:12:04.393;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:12:07.222;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:12:09.536;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:12:21.288;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:12:27.184;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:12:40.448;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:12:41.037;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:12:48.788;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:12:49.485;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:12:52.323;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:12:54.594;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:13:06.359;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:13:12.244;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:13:25.507;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:13:26.107;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:13:33.853;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:13:34.554;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:13:37.394;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:13:39.658;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:13:51.423;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:13:57.305;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:14:10.569;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:14:11.159;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:14:11.179;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:14:18.908;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:14:19.624;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:14:22.465;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:14:24.706;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:14:36.481;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:14:42.366;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:14:55.634;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:14:56.245;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:15:03.970;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:15:04.682;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:15:07.559;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:15:09.786;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:15:21.532;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:15:27.426;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:15:40.698;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:15:41.314;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:15:48.034;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:15:49.750;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:15:52.629;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:15:54.849;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:16:06.598;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:16:12.490;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:16:25.770;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:16:26.380;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:16:33.097;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:16:34.808;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:16:37.689;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:16:39.912;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:16:51.668;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:16:57.553;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:17:10.861;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:17:11.457;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:17:18.162;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:17:19.876;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:17:22.761;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:17:24.967;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:17:36.729;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:17:42.613;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:17:55.925;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:17:56.525;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:18:03.220;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:18:04.921;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:18:07.827;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:18:09.023;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:18:21.793;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:18:27.673;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:18:40.993;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:18:41.595;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:18:48.285;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:18:49.990;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:18:52.893;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:18:54.082;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:19:06.855;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:19:11.254;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:19:12.733;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:19:25.041;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:19:26.656;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:19:33.345;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:19:34.072;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:19:37.941;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:19:39.135;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:19:51.907;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:19:57.789;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:20:10.101;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:20:11.724;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:20:18.405;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:20:19.135;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:20:23.007;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:20:24.180;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:20:36.970;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:20:42.849;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:20:55.161;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:20:56.790;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:21:03.469;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:21:04.204;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:21:07.062;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:21:09.239;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:21:21.034;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:21:27.909;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:21:40.228;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:21:41.856;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:21:48.529;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:21:49.270;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:21:52.119;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:21:54.326;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:22:06.093;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:22:12.970;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:22:25.301;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:22:26.920;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:22:33.587;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:22:34.341;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:22:37.189;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:22:39.384;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:22:51.158;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:22:57.029;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:23:10.360;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:23:11.986;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:23:18.642;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:23:19.412;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:23:22.257;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:23:24.436;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:23:36.233;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:23:42.090;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:23:55.429;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:23:56.045;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:24:03.698;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:24:04.485;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:24:07.369;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:24:09.493;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:24:11.101;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:24:21.293;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:24:27.144;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:24:40.481;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:24:41.120;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:24:48.767;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:24:49.555;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:24:52.438;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:24:54.547;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:25:06.352;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:25:12.204;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:25:25.540;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:25:26.192;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:25:33.828;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:25:34.623;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:25:37.509;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:25:39.609;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:25:51.418;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:25:57.259;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:26:10.600;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:26:11.262;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:26:18.891;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:26:19.691;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:26:22.578;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:26:24.692;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:26:36.482;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:26:42.319;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:26:55.660;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:26:56.327;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:27:03.955;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:27:04.750;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:27:07.647;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:27:09.752;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:27:21.536;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:27:27.379;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:27:40.720;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:27:41.398;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:27:49.010;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:27:49.822;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:27:52.717;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:27:54.813;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:28:06.592;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:28:12.474;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:28:25.781;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:28:26.465;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:28:34.074;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:28:34.893;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:28:37.787;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:28:39.861;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:28:51.657;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:28:57.534;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:29:10.840;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:29:11.196;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:29:11.538;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:29:19.132;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:29:19.964;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:29:22.908;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:29:24.918;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:29:36.713;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:29:42.584;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:29:55.905;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:29:56.606;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:30:04.039;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:30:04.191;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:30:07.980;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:30:09.967;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:30:21.776;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:30:27.639;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:30:40.969;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:30:41.672;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:30:49.105;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:30:49.256;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:30:52.049;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:30:54.014;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:31:06.841;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:31:12.699;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:31:25.033;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:31:26.742;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:31:34.168;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:31:34.321;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:31:37.109;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:31:40.074;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:31:51.908;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:31:57.746;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:32:10.098;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:32:11.813;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:32:19.235;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:32:19.387;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:32:22.178;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:32:25.131;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:32:36.967;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:32:42.800;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:32:55.163;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:32:56.884;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:33:04.306;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:33:04.455;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:33:07.243;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:33:10.188;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:33:21.030;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:33:27.862;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:33:40.220;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:33:41.955;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:33:49.377;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:33:49.518;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:33:52.309;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:33:55.244;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:34:06.092;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:34:11.040;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:34:12.928;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:34:25.280;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:34:26.022;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:34:34.440;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:34:34.579;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:34:37.374;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:34:40.300;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:34:51.150;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:34:57.991;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:35:10.340;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:35:11.078;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:35:19.506;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:35:19.630;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:35:22.443;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:35:25.356;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:35:36.205;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:35:42.051;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:35:55.400;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:35:56.138;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:36:04.570;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:36:04.694;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:36:07.503;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:36:10.413;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:36:21.267;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:36:27.109;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:36:40.459;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:36:41.203;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:36:49.639;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:36:49.759;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:36:52.572;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:36:55.467;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:37:06.332;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:37:12.170;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:37:25.518;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:37:26.273;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:37:34.724;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:37:34.815;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:37:37.641;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:37:40.516;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:37:51.398;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:37:57.231;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:38:10.591;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:38:11.343;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:38:19.796;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:38:19.876;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:38:22.715;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:38:25.576;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:38:36.453;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:38:42.298;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:38:55.652;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:38:56.412;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:39:04.868;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:39:04.929;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:39:07.784;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:39:10.632;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:39:11.135;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:39:21.515;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:39:27.358;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:39:40.718;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:39:41.483;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:39:49.938;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:39:49.992;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:39:52.853;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:39:55.686;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:40:06.573;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:40:12.418;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:40:25.782;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:40:26.551;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:40:34.056;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:40:34.004;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:40:37.923;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:40:40.747;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:40:51.640;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:40:57.479;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:41:10.839;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:41:11.620;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:41:19.069;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:41:19.119;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:41:22.992;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:41:25.800;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:41:36.713;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:41:42.552;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:41:55.903;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:41:56.689;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:42:04.130;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:42:04.183;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:42:07.060;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:42:10.854;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:42:21.780;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:42:27.604;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:42:40.968;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:42:41.760;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:42:49.212;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:42:49.239;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:42:52.129;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:42:55.914;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:43:06.836;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:43:12.664;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:43:25.029;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:43:26.829;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:43:34.283;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:43:34.305;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:43:37.175;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:43:40.967;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:43:51.892;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:43:57.724;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:44:10.088;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:44:11.230;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:44:11.900;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:44:19.341;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:44:19.368;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:44:22.246;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:44:24.015;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:44:36.959;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:44:42.785;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:44:55.154;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:44:56.970;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:45:04.401;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:45:04.428;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:45:07.322;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:45:10.080;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:45:21.022;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:45:27.846;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:45:40.219;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:45:41.024;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:45:49.466;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:45:49.497;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:45:52.386;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:45:55.136;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:46:06.080;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:46:12.910;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:46:25.279;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:46:26.091;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:46:34.527;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:46:34.561;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:46:37.442;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:46:40.191;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:46:51.145;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:46:57.980;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:47:10.339;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:47:11.159;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:47:19.594;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:47:19.619;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:47:22.513;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:47:25.250;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:47:36.201;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:47:42.031;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:47:55.396;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:47:56.229;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:48:04.666;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:48:04.681;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:48:07.576;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:48:10.302;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:48:21.263;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:48:27.089;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:48:40.446;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:48:41.300;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:48:49.736;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:48:49.745;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:48:52.645;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:48:55.359;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:49:06.314;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:49:11.076;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:49:12.160;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:49:25.513;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:49:26.370;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:49:34.806;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:49:34.814;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:49:37.715;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:49:40.416;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:49:51.375;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:49:57.223;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:50:10.578;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:50:11.428;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:50:19.869;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:50:19.876;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:50:22.786;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:50:25.468;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:50:36.443;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:50:42.288;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:50:55.643;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:50:56.498;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:51:04.932;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:51:04.948;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:51:07.854;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:51:10.526;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:51:21.500;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:51:27.350;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:51:40.707;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:51:41.564;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:51:49.997;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:51:50.018;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:51:52.923;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:51:55.584;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:52:06.562;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:52:12.414;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:52:25.774;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:52:26.626;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:52:34.057;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:52:34.086;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:52:37.992;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:52:40.647;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:52:51.628;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:52:57.460;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:53:10.838;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:53:11.695;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:53:19.119;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:53:19.156;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:53:22.060;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:53:25.696;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:53:36.695;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:53:42.524;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:53:55.903;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:53:56.766;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:54:04.183;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:54:04.226;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:54:07.131;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:54:10.747;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:54:11.170;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:54:21.759;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:54:27.590;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:54:40.969;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:54:41.836;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:54:49.239;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:54:49.297;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:54:52.203;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:54:55.802;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:55:06.822;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:55:12.651;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:55:25.021;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:55:26.908;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:55:34.288;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:55:34.362;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:55:37.273;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:55:40.861;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:55:51.889;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:55:57.712;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:56:10.082;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:56:11.977;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:56:19.363;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:56:19.428;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:56:22.337;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:56:25.924;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:56:36.953;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:56:42.774;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:56:55.145;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:56:56.044;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:57:04.427;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:57:04.495;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:57:07.406;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:57:10.976;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:57:22.018;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:57:27.834;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:57:40.207;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:57:41.104;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:57:49.489;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:57:49.561;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:57:52.475;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:57:55.031;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:58:07.065;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:58:12.904;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:58:25.271;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:58:26.174;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:58:34.551;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:58:34.631;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:58:37.546;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:58:40.078;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:58:52.130;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:58:57.956;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:59:10.330;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:59:11.013;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 02:59:11.245;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 02:59:19.609;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 02:59:19.700;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 02:59:22.626;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 02:59:25.142;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 02:59:37.183;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 02:59:42.015;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 02:59:55.394;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 02:59:56.311;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:00:04.673;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:00:04.760;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:00:07.695;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:00:10.197;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:00:22.246;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:00:27.076;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:00:40.459;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:00:41.362;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:00:49.737;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:00:49.829;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:00:52.758;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:00:55.253;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:01:07.303;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:01:12.136;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:01:25.517;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:01:26.422;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:01:34.801;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:01:34.893;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:01:37.823;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:01:40.311;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:01:52.370;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:01:57.196;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:02:10.582;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:02:11.489;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:02:19.859;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:02:19.961;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:02:22.888;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:02:25.367;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:02:37.427;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:02:42.257;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:02:55.654;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:02:56.560;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:03:04.027;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:03:04.920;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:03:07.960;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:03:10.417;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:03:22.482;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:03:27.324;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:03:40.710;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:03:41.631;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:03:49.094;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:03:49.982;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:03:52.030;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:03:55.471;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:04:07.534;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:04:11.107;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:04:12.385;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:04:25.802;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:04:26.701;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:04:34.036;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:04:34.163;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:04:37.096;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:04:40.537;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:04:52.592;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:04:57.445;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:05:10.860;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:05:11.769;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:05:19.094;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:05:19.229;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:05:22.178;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:05:25.606;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:05:37.657;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:05:42.505;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:05:55.925;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:05:56.837;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:06:04.156;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:06:04.301;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:06:07.246;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:06:10.663;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:06:22.715;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:06:27.568;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:06:40.990;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:06:41.905;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:06:49.210;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:06:49.372;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:06:52.313;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:06:55.722;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:07:07.786;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:07:12.628;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:07:25.056;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:07:26.973;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:07:34.265;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:07:34.442;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:07:37.383;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:07:40.778;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:07:52.851;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:07:57.681;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:08:10.116;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:08:11.034;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:08:19.327;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:08:19.476;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:08:22.446;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:08:25.836;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:08:37.917;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:08:42.745;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:08:55.177;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:08:56.105;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:09:04.392;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:09:04.540;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:09:07.523;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:09:10.887;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:09:11.201;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:09:22.021;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:09:27.807;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:09:40.239;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:09:41.174;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:09:49.449;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:09:49.613;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:09:52.593;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:09:55.936;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:10:07.119;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:10:12.874;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:10:25.308;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:10:26.245;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:10:34.501;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:10:34.683;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:10:37.661;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:10:40.981;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:10:52.186;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:10:57.972;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:11:10.375;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:11:11.317;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:11:19.568;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:11:19.748;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:11:22.727;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:11:25.041;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:11:37.299;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:11:42.031;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:11:55.441;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:11:56.375;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:12:04.630;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:12:04.823;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:12:07.797;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:12:10.098;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:12:22.366;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:12:27.067;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:12:40.501;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:12:41.447;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:12:49.691;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:12:49.894;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:12:52.868;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:12:55.155;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:13:07.421;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:13:12.123;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:13:25.557;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:13:26.514;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:13:34.754;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:13:34.964;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:13:37.938;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:13:40.207;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:13:52.495;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:13:57.183;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:14:10.620;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:14:11.045;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:14:11.587;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:14:19.032;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:14:19.819;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:14:22.009;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:14:25.290;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:14:37.552;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:14:42.244;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:14:55.707;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:14:56.653;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:15:04.098;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:15:04.880;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:15:07.078;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:15:10.347;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:15:22.602;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:15:27.304;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:15:40.773;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:15:41.719;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:15:49.165;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:15:49.942;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:15:52.148;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:15:55.403;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:16:07.669;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:16:12.364;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:16:25.840;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:16:26.784;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:16:34.234;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:16:35.007;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:16:37.207;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:16:40.463;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:16:52.728;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:16:57.425;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:17:10.908;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:17:11.850;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:17:19.079;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:17:19.301;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:17:22.277;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:17:25.519;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:17:37.790;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:17:42.485;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:17:55.967;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:17:56.921;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:18:04.140;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:18:04.364;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:18:07.344;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:18:10.576;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:18:22.853;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:18:27.544;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:18:40.028;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:18:41.987;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:18:49.209;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:18:49.434;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:18:52.406;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:18:55.627;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:19:07.924;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:19:11.135;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:19:12.606;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:19:25.091;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:19:26.057;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:19:34.281;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:19:34.505;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:19:37.477;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:19:40.679;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:19:52.981;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:19:57.666;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:20:10.160;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:20:11.121;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:20:19.342;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:20:19.565;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:20:22.547;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:20:25.754;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:20:37.046;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:20:42.727;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:20:55.220;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:20:56.188;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:21:04.399;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:21:04.619;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:21:07.611;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:21:10.811;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:21:22.106;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:21:27.779;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:21:40.282;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:21:41.255;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:21:49.465;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:21:49.688;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:21:52.678;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:21:55.861;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:22:07.172;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:22:12.841;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:22:25.346;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:22:26.317;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:22:34.524;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:22:34.759;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:22:37.743;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:22:40.918;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:22:52.232;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:22:57.901;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:23:10.403;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:23:11.389;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:23:19.588;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:23:19.832;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:23:22.814;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:23:25.968;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:23:37.293;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:23:42.959;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:23:55.466;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:23:56.455;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:24:04.651;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:24:04.903;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:24:07.894;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:24:10.026;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:24:11.224;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:24:22.359;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:24:27.019;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:24:40.542;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:24:41.526;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:24:49.713;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:24:49.972;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:24:52.991;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:24:55.075;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:25:07.422;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:25:12.080;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:25:25.607;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:25:26.592;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:25:34.030;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:25:34.777;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:25:37.059;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:25:40.132;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:25:52.496;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:25:57.140;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:26:10.668;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:26:11.699;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:26:19.095;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:26:19.842;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:26:22.128;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:26:25.187;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:26:37.551;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:26:42.200;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:26:55.727;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:26:56.767;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:27:04.152;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:27:04.892;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:27:07.200;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:27:10.242;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:27:22.614;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:27:27.254;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:27:40.792;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:27:41.838;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:27:49.222;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:27:49.948;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:27:52.267;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:27:55.299;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:28:07.673;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:28:12.313;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:28:25.849;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:28:26.908;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:28:34.296;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:28:35.012;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:28:37.338;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:28:40.341;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:28:52.732;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:28:57.373;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:29:10.914;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:29:11.065;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:29:12.015;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:29:19.366;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:29:20.076;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:29:22.408;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:29:25.398;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:29:37.794;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:29:42.424;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:29:55.977;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:29:56.083;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:30:04.432;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:30:05.130;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:30:07.506;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:30:10.446;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:30:22.857;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:30:27.485;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:30:40.039;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:30:41.150;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:30:49.491;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:30:50.195;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:30:52.575;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:30:55.503;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:31:07.924;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:31:12.545;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:31:25.103;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:31:26.215;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:31:34.548;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:31:35.259;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:31:37.638;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:31:40.560;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:31:52.982;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:31:57.603;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:32:10.169;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:32:11.279;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:32:19.611;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:32:20.319;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:32:22.707;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:32:25.617;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:32:37.043;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:32:42.667;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:32:55.227;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:32:56.349;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:33:04.679;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:33:05.380;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:33:07.771;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:33:10.673;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:33:22.105;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:33:27.727;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:33:40.290;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:33:41.420;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:33:49.748;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:33:50.446;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:33:52.839;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:33:55.760;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:34:07.168;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:34:11.156;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:34:12.787;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:34:25.347;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:34:26.486;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:34:34.819;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:34:35.506;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:34:37.915;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:34:40.817;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:34:52.229;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:34:57.869;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:35:10.412;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:35:11.552;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:35:19.879;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:35:20.565;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:35:22.985;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:35:25.871;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:35:37.292;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:35:42.929;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:35:55.477;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:35:56.619;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:36:04.951;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:36:05.614;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:36:07.050;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:36:10.929;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:36:22.357;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:36:27.990;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:36:40.540;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:36:41.687;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:36:49.018;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:36:50.670;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:36:52.112;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:36:55.977;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:37:07.421;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:37:12.051;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:37:25.605;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:37:26.746;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:37:34.075;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:37:35.725;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:37:37.173;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:37:40.036;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:37:52.482;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:37:57.110;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:38:10.673;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:38:11.806;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:38:19.135;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:38:20.788;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:38:22.245;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:38:25.086;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:38:37.536;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:38:42.170;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:38:55.727;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:38:56.867;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:39:04.205;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:39:05.852;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:39:07.314;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:39:10.153;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:39:11.250;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:39:22.602;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:39:27.228;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:39:40.790;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:39:41.937;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:39:49.274;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:39:50.909;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:39:52.385;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:39:55.211;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:40:07.663;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:40:12.287;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:40:25.848;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:40:26.004;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:40:34.343;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:40:35.974;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:40:37.451;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:40:40.268;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:40:52.721;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:40:57.347;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:41:10.911;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:41:11.072;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:41:19.409;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:41:20.041;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:41:22.514;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:41:25.328;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:41:37.789;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:41:42.399;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:41:55.977;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:41:56.137;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:42:04.475;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:42:05.099;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:42:07.580;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:42:10.385;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:42:22.842;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:42:27.455;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:42:40.040;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:42:41.206;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:42:49.534;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:42:50.160;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:42:52.651;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:42:55.441;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:43:07.908;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:43:12.515;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:43:25.109;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:43:26.270;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:43:34.603;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:43:35.218;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:43:37.722;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:43:40.508;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:43:52.972;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:43:57.574;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:44:10.174;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:44:11.090;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:44:11.366;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:44:19.674;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:44:20.283;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:44:22.774;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:44:25.556;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:44:37.023;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:44:42.630;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:44:55.240;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:44:56.432;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:45:04.739;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:45:05.351;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:45:07.844;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:45:10.609;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:45:22.090;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:45:27.692;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:45:40.305;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:45:41.498;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:45:49.806;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:45:50.409;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:45:52.914;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:45:55.668;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:46:07.157;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:46:12.752;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:46:25.364;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:46:26.563;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:46:34.871;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:46:35.469;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:46:37.982;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:46:40.728;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:46:52.223;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:46:57.824;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:47:10.429;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:47:11.628;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:47:19.913;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:47:20.529;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:47:22.051;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:47:25.787;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:47:37.285;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:47:42.904;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:47:55.487;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:47:56.699;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:48:04.974;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:48:05.591;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:48:07.116;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:48:10.853;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:48:22.355;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:48:27.970;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:48:40.538;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:48:41.771;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:48:49.043;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:48:50.654;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:48:52.192;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:48:55.910;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:49:07.419;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:49:11.181;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:49:12.030;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:49:25.603;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:49:26.865;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:49:34.114;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:49:35.714;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:49:37.261;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:49:40.957;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:49:52.481;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:49:57.092;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:50:10.669;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:50:11.938;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:50:19.179;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:50:20.776;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:50:22.331;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:50:26.007;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:50:37.536;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:50:42.152;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:50:55.727;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:50:56.004;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:51:04.244;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:51:05.842;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:51:07.398;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:51:11.064;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:51:22.607;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:51:27.213;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:51:40.790;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:51:41.062;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:51:49.313;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:51:50.896;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:51:52.467;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:51:56.125;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:52:07.671;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:52:12.266;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:52:25.848;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:52:26.127;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:52:34.382;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:52:35.955;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:52:37.537;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:52:41.178;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:52:52.732;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:52:57.311;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:53:10.903;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:53:11.217;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:53:19.454;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:53:20.016;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:53:22.599;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:53:26.236;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:53:37.798;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:53:42.371;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:53:55.967;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:53:56.277;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:54:04.526;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:54:05.080;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:54:07.670;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:54:11.022;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:54:11.293;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:54:22.862;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:54:27.437;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:54:40.030;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:54:41.347;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:54:49.596;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:54:50.133;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:54:52.740;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:54:56.352;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:55:07.921;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:55:12.500;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:55:25.096;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:55:26.414;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:55:34.660;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:55:35.190;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:55:37.809;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:55:41.410;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:55:52.983;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:55:57.562;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:56:10.157;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:56:11.484;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:56:19.731;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:56:20.255;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:56:22.876;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:56:26.468;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:56:37.074;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:56:42.619;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:56:55.218;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:56:56.565;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:57:04.803;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:57:05.319;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:57:07.937;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:57:11.517;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:57:22.139;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:57:27.682;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:57:40.285;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:57:41.662;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:57:49.876;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:57:50.379;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:57:53.010;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:57:56.574;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:58:07.204;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:58:12.744;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:58:25.339;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:58:26.732;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:58:34.957;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:58:35.444;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:58:37.078;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:58:41.627;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:58:52.266;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:58:57.805;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:59:10.404;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:59:11.115;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 03:59:11.802;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 03:59:19.026;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 03:59:20.508;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 03:59:22.149;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 03:59:26.679;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 03:59:37.323;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 03:59:42.866;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 03:59:55.478;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 03:59:56.873;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:00:04.081;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:00:05.572;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:00:07.219;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:00:11.736;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:00:22.386;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:00:27.931;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:00:40.540;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:00:41.935;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:00:49.135;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:00:50.630;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:00:52.288;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:00:56.793;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:01:07.451;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:01:12.991;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:01:25.605;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:01:27.038;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:01:34.202;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:01:35.694;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:01:37.356;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:01:41.848;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:01:52.535;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:01:57.050;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:02:10.671;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:02:12.103;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:02:19.270;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:02:20.756;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:02:22.427;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:02:26.904;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:02:37.587;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:02:42.111;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:02:55.728;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:02:57.163;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:03:04.358;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:03:05.819;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:03:07.495;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:03:11.958;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:03:22.649;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:03:27.171;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:03:40.812;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:03:42.232;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:03:49.425;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:03:50.871;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:03:52.554;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:03:56.015;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:04:07.707;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:04:11.209;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:04:12.234;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:04:25.870;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:04:27.301;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:04:34.497;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:04:35.927;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:04:37.623;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:04:41.071;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:04:52.820;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:04:57.294;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:05:10.936;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:05:12.374;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:05:19.555;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:05:20.988;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:05:22.711;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:05:26.127;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:05:37.886;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:05:42.355;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:05:55.002;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:05:57.444;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:06:04.624;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:06:05.052;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:06:07.771;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:06:11.179;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:06:22.952;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:06:27.409;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:06:40.066;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:06:42.504;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:06:49.692;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:06:50.108;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:06:52.839;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:06:56.227;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:07:07.016;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:07:12.465;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:07:25.133;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:07:27.566;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:07:34.761;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:07:35.166;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:07:37.897;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:07:41.282;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:07:52.107;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:07:57.525;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:08:10.193;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:08:12.634;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:08:19.833;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:08:20.231;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:08:22.966;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:08:26.338;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:08:37.172;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:08:42.582;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:08:55.257;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:08:57.710;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:09:04.910;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:09:05.293;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:09:07.040;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:09:11.058;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:09:11.385;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:09:22.234;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:09:27.643;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:09:40.318;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:09:42.775;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:09:49.984;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:09:50.353;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:09:52.110;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:09:56.437;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:10:07.294;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:10:12.713;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:10:25.387;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:10:27.845;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:10:34.053;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:10:35.411;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:10:37.177;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:10:41.495;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:10:52.351;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:10:57.775;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:11:10.451;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:11:12.931;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:11:19.121;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:11:20.474;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:11:22.246;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:11:26.548;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:11:37.417;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:11:42.829;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:11:55.511;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:11:57.998;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:12:04.188;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:12:05.529;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:12:07.306;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:12:11.604;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:12:22.479;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:12:27.884;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:12:40.576;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:12:42.064;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:12:49.252;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:12:50.589;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:12:52.377;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:12:56.661;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:13:07.532;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:13:12.943;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:13:25.640;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:13:27.133;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:13:34.322;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:13:35.648;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:13:37.447;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:13:41.708;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:13:52.597;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:13:57.034;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:14:10.705;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:14:11.155;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:14:12.210;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:14:19.394;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:14:20.701;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:14:22.517;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:14:26.767;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:14:37.652;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:14:42.093;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:14:55.757;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:14:57.275;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:15:04.461;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:15:05.766;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:15:07.580;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:15:11.828;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:15:22.713;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:15:27.144;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:15:40.823;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:15:42.344;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:15:49.528;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:15:50.833;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:15:52.659;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:15:56.874;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:16:07.779;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:16:12.195;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:16:25.880;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:16:27.401;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:16:34.586;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:16:35.891;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:16:37.717;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:16:41.929;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:16:52.844;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:16:57.254;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:17:10.946;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:17:12.467;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:17:19.653;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:17:20.956;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:17:22.786;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:17:26.978;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:17:37.911;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:17:42.315;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:17:55.013;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:17:57.536;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:18:04.723;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:18:05.019;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:18:07.852;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:18:11.034;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:18:22.968;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:18:27.375;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:18:41.071;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:18:42.606;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:18:49.792;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:18:50.083;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:18:52.919;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:18:56.090;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:19:07.023;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:19:11.255;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:19:12.434;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:19:26.133;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:19:27.668;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:19:34.862;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:19:35.137;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:19:37.989;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:19:41.146;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:19:52.085;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:19:57.483;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:20:11.188;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:20:12.732;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:20:19.925;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:20:20.201;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:20:22.057;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:20:26.198;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:20:37.147;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:20:42.533;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:20:56.252;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:20:57.802;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:21:04.993;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:21:05.265;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:21:07.120;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:21:11.247;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:21:22.205;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:21:27.608;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:21:41.318;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:21:42.870;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:21:49.061;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:21:50.320;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:21:52.150;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:21:56.298;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:22:07.267;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:22:12.667;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:22:26.381;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:22:27.938;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:22:34.125;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:22:35.384;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:22:37.218;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:22:41.356;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:22:52.332;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:22:57.727;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:23:11.441;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:23:12.006;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:23:19.197;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:23:20.441;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:23:22.287;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:23:26.414;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:23:37.397;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:23:42.789;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:23:56.500;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:23:57.073;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:24:04.263;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:24:05.504;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:24:07.358;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:24:11.105;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:24:11.469;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:24:22.456;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:24:27.871;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:24:41.564;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:24:42.136;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:24:49.328;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:24:50.569;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:24:52.427;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:24:56.517;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:25:07.505;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:25:12.933;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:25:26.628;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:25:27.202;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:25:34.391;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:25:35.629;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:25:37.497;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:25:41.577;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:25:52.563;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:25:57.995;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:26:11.690;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:26:12.269;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:26:19.460;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:26:20.693;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:26:22.555;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:26:26.650;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:26:37.626;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:26:42.054;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:26:56.749;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:26:57.335;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:27:04.525;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:27:05.758;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:27:07.627;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:27:11.698;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:27:22.678;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:27:27.115;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:27:41.812;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:27:42.403;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:27:49.596;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:27:50.821;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:27:52.697;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:27:56.756;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:28:07.732;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:28:12.175;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:28:26.871;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:28:27.461;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:28:34.705;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:28:35.880;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:28:37.769;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:28:41.821;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:28:52.796;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:28:57.235;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:29:11.197;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:29:11.936;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:29:12.531;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:29:19.772;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:29:20.946;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:29:22.838;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:29:26.873;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:29:37.856;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:29:42.295;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:29:57.007;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:29:57.593;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:30:04.830;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:30:06.009;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:30:07.908;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:30:11.929;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:30:22.919;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:30:27.357;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:30:41.074;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:30:42.655;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:30:49.899;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:30:51.083;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:30:52.977;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:30:56.980;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:31:07.982;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:31:12.417;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:31:26.137;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:31:27.725;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:31:34.968;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:31:36.138;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:31:37.061;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:31:41.037;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:31:52.045;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:31:57.477;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:32:11.212;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:32:12.795;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:32:19.044;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:32:21.201;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:32:22.126;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:32:26.093;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:32:37.102;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:32:42.534;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:32:56.277;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:32:57.863;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:33:04.112;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:33:06.269;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:33:07.192;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:33:11.148;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:33:22.156;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:33:27.593;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:33:41.339;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:33:42.933;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:33:49.211;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:33:51.334;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:33:52.258;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:33:56.197;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:34:07.206;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:34:11.040;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:34:12.639;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:34:26.403;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:34:27.002;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:34:34.279;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:34:36.390;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:34:37.330;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:34:41.255;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:34:52.261;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:34:57.698;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:35:11.466;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:35:12.064;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:35:19.336;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:35:21.454;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:35:22.400;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:35:26.312;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:35:37.326;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:35:42.752;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:35:56.529;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:35:57.124;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:36:04.402;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:36:06.519;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:36:07.471;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:36:11.374;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:36:22.391;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:36:27.810;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:36:41.597;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:36:42.194;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:36:49.460;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:36:51.583;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:36:52.540;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:36:56.428;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:37:07.456;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:37:12.871;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:37:26.685;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:37:27.256;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:37:34.527;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:37:36.637;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:37:37.602;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:37:41.484;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:37:52.517;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:37:57.931;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:38:11.745;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:38:12.324;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:38:19.602;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:38:21.698;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:38:22.694;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:38:26.550;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:38:37.573;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:38:42.992;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:38:56.806;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:38:57.388;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:39:04.662;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:39:06.762;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:39:07.767;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:39:11.132;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:39:11.606;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:39:22.638;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:39:27.053;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:39:41.872;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:39:42.458;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:39:49.731;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:39:51.826;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:39:52.836;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:39:56.666;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:40:07.702;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:40:12.114;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:40:26.928;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:40:27.527;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:40:34.789;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:40:36.889;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:40:37.907;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:40:41.718;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:40:52.768;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:40:57.174;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:41:11.986;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:41:12.596;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:41:19.896;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:41:21.948;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:41:23.005;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:41:26.771;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:41:37.826;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:41:42.234;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:41:56.049;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:41:57.663;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:42:04.962;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:42:07.013;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:42:07.094;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:42:11.834;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:42:22.902;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:42:27.293;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:42:41.128;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:42:42.731;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:42:49.029;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:42:52.079;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:42:52.167;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:42:56.884;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:43:07.969;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:43:12.349;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:43:26.188;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:43:27.801;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:43:34.098;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:43:37.135;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:43:37.237;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:43:41.940;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:43:52.034;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:43:57.408;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:44:11.225;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:44:11.251;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:44:12.870;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:44:19.162;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:44:22.200;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:44:22.308;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:44:26.988;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:44:37.110;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:44:42.458;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:44:56.311;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:44:57.938;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:45:04.268;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:45:07.264;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:45:07.379;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:45:11.045;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:45:22.179;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:45:27.519;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:45:41.369;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:45:42.002;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:45:49.332;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:45:52.331;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:45:52.448;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:45:56.096;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:46:07.243;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:46:12.573;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:46:26.429;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:46:27.066;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:46:34.394;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:46:37.390;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:46:37.507;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:46:41.154;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:46:52.306;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:46:57.633;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:47:11.478;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:47:12.127;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:47:19.460;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:47:22.449;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:47:22.605;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:47:26.209;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:47:37.368;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:47:42.704;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:47:56.541;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:47:57.196;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:48:04.530;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:48:07.505;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:48:07.714;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:48:11.257;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:48:22.427;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:48:27.765;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:48:41.600;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:48:42.265;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:48:49.591;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:48:52.569;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:48:52.779;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:48:56.307;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:49:07.481;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:49:11.067;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:49:12.825;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:49:26.665;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:49:27.333;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:49:34.661;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:49:37.627;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:49:37.852;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:49:41.363;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:49:52.536;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:49:57.885;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:50:11.729;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:50:12.399;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:50:19.725;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:50:22.690;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:50:22.915;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:50:26.427;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:50:37.602;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:50:42.945;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:50:56.793;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:50:57.466;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:51:04.791;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:51:07.754;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:51:07.990;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:51:11.484;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:51:22.663;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:51:28.006;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:51:41.848;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:51:42.533;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:51:49.861;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:51:52.056;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:51:52.818;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:51:56.542;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:52:07.722;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:52:12.065;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:52:26.903;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:52:27.600;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:52:34.930;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:52:37.125;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:52:37.875;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:52:41.599;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:52:52.787;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:52:57.125;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:53:11.955;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:53:12.668;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:53:19.000;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:53:22.194;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:53:22.936;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:53:26.658;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:53:37.852;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:53:42.203;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:53:56.015;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:53:57.737;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:54:04.072;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:54:07.283;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:54:07.000;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:54:11.156;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:54:11.707;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:54:22.918;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:54:27.268;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:54:41.080;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:54:42.806;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:54:49.142;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:54:52.063;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:54:52.370;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:54:56.763;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:55:07.984;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:55:12.328;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:55:26.139;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:55:27.874;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:55:34.209;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:55:37.119;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:55:37.441;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:55:41.820;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:55:52.047;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:55:57.388;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:56:11.198;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:56:12.944;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:56:19.308;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:56:22.182;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:56:22.506;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:56:26.880;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:56:37.112;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:56:42.444;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:56:56.255;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:56:57.010;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:57:04.373;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:57:07.246;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:57:07.568;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:57:11.927;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:57:22.170;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:57:27.503;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:57:41.321;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:57:43.088;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:57:49.438;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:57:52.309;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:57:52.649;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:57:56.973;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:58:07.233;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:58:12.563;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:58:26.382;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:58:28.145;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:58:34.502;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:58:37.370;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:58:37.719;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:58:41.036;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:58:52.293;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:58:57.622;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:59:11.247;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 04:59:11.446;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:59:13.216;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 04:59:19.571;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 04:59:22.435;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 04:59:22.819;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 04:59:26.085;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 04:59:37.352;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 04:59:42.684;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 04:59:56.514;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 04:59:58.281;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:00:04.636;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:00:07.490;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:00:07.915;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:00:11.140;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:00:22.416;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:00:27.745;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:00:41.568;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:00:43.345;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:00:49.701;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:00:52.556;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:00:52.992;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:00:56.198;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:01:07.471;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:01:12.804;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:01:26.632;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:01:28.408;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:01:34.731;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:01:37.052;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:01:37.619;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:01:41.248;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:01:52.532;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:01:57.864;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:02:11.701;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:02:13.471;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:02:19.801;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:02:22.114;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:02:22.679;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:02:26.297;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:02:37.588;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:02:42.924;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:02:56.760;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:02:58.547;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:03:04.870;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:03:07.181;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:03:07.736;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:03:11.340;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:03:22.651;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:03:27.986;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:03:41.820;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:03:43.616;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:03:49.939;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:03:52.248;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:03:52.809;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:03:56.397;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:04:07.715;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:04:11.087;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:04:12.045;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:04:26.884;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:04:28.684;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:04:34.005;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:04:37.316;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:04:37.866;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:04:41.447;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:04:52.770;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:04:57.105;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:05:11.949;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:05:13.753;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:05:19.068;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:05:22.384;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:05:22.921;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:05:26.503;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:05:37.835;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:05:42.164;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:05:57.052;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:05:58.818;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:06:04.148;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:06:07.441;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:06:07.987;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:06:11.560;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:06:22.900;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:06:27.218;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:06:42.120;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:06:43.879;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:06:49.220;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:06:52.050;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:06:52.509;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:06:56.609;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:07:07.956;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:07:12.281;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:07:27.180;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:07:28.948;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:07:34.289;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:07:37.121;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:07:37.576;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:07:41.677;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:07:52.018;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:07:57.342;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:08:12.247;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:08:13.013;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:08:19.361;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:08:22.185;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:08:22.613;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:08:26.734;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:08:37.084;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:08:42.401;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:08:57.336;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:08:59.076;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:09:04.419;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:09:07.249;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:09:07.687;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:09:11.179;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:09:11.791;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:09:22.148;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:09:27.460;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:09:42.399;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:09:44.147;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:09:49.500;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:09:52.314;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:09:52.756;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:09:56.850;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:10:07.202;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:10:12.521;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:10:27.461;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:10:29.217;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:10:34.571;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:10:37.369;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:10:37.830;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:10:41.906;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:10:52.263;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:10:57.582;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:11:12.516;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:11:14.308;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:11:19.640;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:11:22.428;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:11:22.885;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:11:26.959;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:11:37.329;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:11:42.638;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:11:57.570;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:11:59.377;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:12:04.711;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:12:07.493;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:12:07.954;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:12:12.012;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:12:22.391;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:12:27.699;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:12:42.631;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:12:44.447;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:12:49.780;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:12:52.025;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:12:52.549;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:12:57.072;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:13:07.454;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:13:12.760;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:13:27.689;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:13:29.518;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:13:34.850;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:13:37.098;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:13:37.608;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:13:42.128;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:13:52.508;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:13:57.821;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:14:11.020;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:14:12.750;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:14:14.586;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:14:19.922;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:14:22.165;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:14:22.672;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:14:27.178;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:14:37.562;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:14:42.888;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:14:57.816;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:14:59.652;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:15:04.987;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:15:07.234;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:15:07.737;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:15:12.225;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:15:22.626;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:15:27.947;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:15:42.881;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:15:44.717;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:15:49.058;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:15:52.305;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:15:52.802;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:15:57.282;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:16:07.690;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:16:13.007;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:16:27.955;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:16:29.780;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:16:34.124;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:16:37.367;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:16:37.858;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:16:42.345;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:16:52.756;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:16:57.097;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:17:13.010;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:17:14.851;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:17:19.196;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:17:22.439;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:17:22.917;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:17:27.402;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:17:37.817;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:17:42.159;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:17:58.076;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:17:59.927;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:18:04.267;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:18:07.507;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:18:07.983;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:18:12.460;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:18:22.868;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:18:27.218;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:18:43.135;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:18:44.996;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:18:49.336;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:18:52.055;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:18:52.567;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:18:57.509;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:19:07.932;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:19:11.110;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:19:12.280;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:19:28.201;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:19:29.064;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:19:34.406;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:19:37.115;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:19:37.630;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:19:42.566;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:19:52.994;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:19:57.340;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:20:13.260;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:20:14.130;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:20:19.472;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:20:22.169;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:20:22.700;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:20:27.624;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:20:37.057;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:20:42.401;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:20:58.325;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:20:59.197;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:21:04.550;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:21:07.223;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:21:07.757;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:21:12.708;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:21:22.121;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:21:27.461;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:21:43.381;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:21:44.257;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:21:49.622;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:21:52.282;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:21:52.818;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:21:57.766;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:22:07.186;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:22:12.525;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:22:28.439;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:22:29.331;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:22:34.692;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:22:37.341;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:22:37.888;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:22:42.824;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:22:52.243;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:22:57.584;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:23:13.489;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:23:14.411;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:23:19.766;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:23:22.398;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:23:22.957;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:23:27.872;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:23:37.301;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:23:42.644;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:23:58.574;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:23:59.480;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:24:04.851;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:24:07.026;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:24:07.463;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:24:11.202;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:24:12.929;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:24:22.366;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:24:27.708;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:24:43.639;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:24:44.549;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:24:49.920;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:24:52.096;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:24:52.518;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:24:57.988;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:25:07.422;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:25:12.768;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:25:28.703;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:25:29.617;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:25:34.980;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:25:37.163;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:25:37.581;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:25:42.037;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:25:52.482;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:25:57.828;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:26:13.757;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:26:14.684;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:26:19.045;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:26:22.234;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:26:22.640;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:26:27.101;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:26:37.536;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:26:42.889;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:26:58.818;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:26:59.746;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:27:04.113;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:27:07.304;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:27:07.699;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:27:12.158;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:27:22.600;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:27:27.920;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:27:43.912;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:27:44.814;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:27:49.181;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:27:52.373;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:27:52.759;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:27:57.200;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:28:07.662;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:28:12.974;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:28:28.972;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:28:29.882;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:28:34.249;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:28:37.443;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:28:37.818;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:28:42.248;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:28:52.716;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:28:57.038;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:29:11.046;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:29:13.035;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:29:14.946;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:29:19.319;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:29:22.514;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:29:22.879;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:29:27.318;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:29:37.763;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:29:42.122;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:29:58.100;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:30:00.012;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:30:04.387;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:30:07.579;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:30:07.943;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:30:12.380;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:30:22.827;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:30:27.183;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:30:43.164;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:30:45.077;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:30:49.450;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:30:52.646;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:30:52.007;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:30:57.437;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:31:07.892;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:31:12.270;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:31:28.229;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:31:30.142;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:31:34.515;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:31:37.070;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:31:37.696;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:31:42.502;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:31:52.956;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:31:57.334;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:32:13.293;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:32:15.206;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:32:19.582;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:32:22.129;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:32:22.766;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:32:27.560;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:32:37.018;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:32:42.427;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:32:58.350;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:33:00.278;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:33:04.651;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:33:07.182;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:33:07.832;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:33:12.617;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:33:22.080;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:33:27.492;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:33:43.409;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:33:45.347;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:33:49.713;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:33:52.246;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:33:52.893;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:33:57.674;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:34:07.141;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:34:11.138;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:34:12.548;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:34:28.468;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:34:30.416;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:34:34.782;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:34:37.296;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:34:37.966;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:34:42.731;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:34:52.198;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:34:57.643;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:35:13.531;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:35:15.481;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:35:19.846;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:35:22.035;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:35:22.357;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:35:27.787;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:35:37.263;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:35:42.707;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:35:58.591;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:36:00.546;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:36:04.915;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:36:07.099;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:36:07.423;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:36:12.845;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:36:22.329;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:36:27.803;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:36:43.656;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:36:45.603;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:36:49.985;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:36:52.163;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:36:52.487;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:36:57.898;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:37:07.394;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:37:12.879;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:37:28.724;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:37:30.668;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:37:34.052;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:37:37.222;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:37:37.550;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:37:42.947;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:37:52.456;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:37:57.941;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:38:13.786;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:38:15.741;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:38:19.122;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:38:22.312;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:38:22.609;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:38:27.000;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:38:37.515;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:38:42.021;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:38:58.850;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:39:00.811;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:39:04.169;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:39:07.387;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:39:07.674;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:39:11.237;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:39:12.056;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:39:22.575;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:39:27.084;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:39:43.910;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:39:45.881;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:39:49.238;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:39:52.459;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:39:52.738;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:39:57.107;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:40:07.620;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:40:12.138;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:40:28.961;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:40:30.953;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:40:34.302;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:40:37.527;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:40:37.795;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:40:42.166;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:40:52.682;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:40:57.202;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:41:13.021;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:41:16.016;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:41:19.385;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:41:22.594;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:41:22.879;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:41:27.229;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:41:37.741;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:41:42.259;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:41:58.085;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:42:00.077;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:42:04.451;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:42:07.661;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:42:07.944;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:42:12.292;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:42:22.802;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:42:27.319;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:42:43.149;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:42:45.147;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:42:49.512;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:42:52.024;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:42:52.744;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:42:57.349;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:43:07.864;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:43:12.380;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:43:28.210;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:43:30.215;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:43:34.586;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:43:37.089;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:43:37.821;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:43:42.399;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:43:52.920;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:43:57.456;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:44:11.088;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:44:13.277;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:44:15.285;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:44:19.661;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:44:22.150;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:44:22.889;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:44:27.447;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:44:37.971;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:44:42.536;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:44:58.343;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:45:00.351;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:45:04.719;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:45:07.208;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:45:07.961;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:45:12.504;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:45:22.034;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:45:27.598;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:45:43.409;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:45:45.414;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:45:49.808;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:45:52.034;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:45:52.271;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:45:57.560;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:46:07.098;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:46:12.659;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:46:28.468;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:46:30.491;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:46:34.875;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:46:37.097;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:46:37.335;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:46:42.609;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:46:52.162;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:46:57.730;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:47:13.530;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:47:15.551;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:47:19.940;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:47:22.170;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:47:22.380;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:47:27.666;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:47:37.220;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:47:42.794;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:47:58.595;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:48:00.620;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:48:04.010;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:48:07.238;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:48:07.444;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:48:12.719;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:48:22.281;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:48:27.871;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:48:43.651;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:48:45.688;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:48:49.090;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:48:52.305;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:48:52.509;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:48:57.777;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:49:07.344;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:49:11.183;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:49:12.934;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:49:28.710;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:49:30.758;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:49:34.162;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:49:37.375;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:49:37.571;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:49:42.834;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:49:52.394;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:49:57.988;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:50:13.768;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:50:15.821;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:50:19.226;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:50:22.445;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:50:22.641;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:50:27.883;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:50:37.459;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:50:42.047;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:50:58.824;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:51:00.887;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:51:04.282;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:51:07.523;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:51:07.705;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:51:12.938;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:51:22.516;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:51:27.113;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:51:43.881;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:51:45.955;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:51:49.371;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:51:52.589;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:51:52.778;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:51:57.989;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:52:07.572;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:52:12.168;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:52:28.946;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:52:30.024;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:52:34.440;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:52:37.652;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:52:37.846;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:52:42.045;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:52:52.635;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:52:57.230;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:53:13.009;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:53:15.092;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:53:19.512;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:53:22.721;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:53:22.910;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:53:27.110;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:53:37.699;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:53:42.292;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:53:59.072;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:54:00.164;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:54:04.591;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:54:07.801;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:54:07.974;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:54:11.024;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:54:12.166;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:54:22.764;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:54:27.356;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:54:44.136;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:54:45.236;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:54:49.660;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:54:52.036;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:54:52.873;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:54:57.216;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:55:07.821;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:55:12.419;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:55:29.200;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:55:30.303;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:55:34.720;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:55:37.098;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:55:37.944;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:55:42.267;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:55:52.886;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:55:57.480;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:56:14.257;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:56:15.365;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:56:19.784;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:56:22.159;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:56:22.006;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:56:27.327;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:56:37.950;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:56:42.534;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:56:59.319;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:57:00.433;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:57:04.851;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:57:07.095;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:57:07.223;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:57:12.387;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:57:22.004;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:57:27.613;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:57:44.383;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:57:45.501;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:57:49.921;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:57:52.165;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:57:52.279;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:57:57.437;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:58:07.067;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:58:12.674;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:58:29.441;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:58:30.569;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:58:34.995;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:58:37.237;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:58:37.345;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:58:42.494;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:58:52.131;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:58:57.736;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:59:11.117;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 05:59:14.492;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 05:59:15.640;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 05:59:19.062;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 05:59:22.310;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 05:59:22.409;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 05:59:27.548;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 05:59:37.181;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 05:59:42.812;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 05:59:59.550;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:00:00.706;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:00:04.117;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:00:07.382;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:00:07.468;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:00:12.604;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:00:22.230;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:00:27.873;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:00:44.606;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:00:45.771;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:00:49.184;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:00:52.466;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:00:52.528;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:00:57.660;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:01:07.293;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:01:12.935;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:01:29.670;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:01:30.835;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:01:34.245;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:01:37.529;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:01:37.591;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:01:42.707;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:01:52.360;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:01:57.988;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:02:14.730;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:02:15.891;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:02:19.311;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:02:22.600;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:02:22.662;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:02:27.764;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:02:37.420;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:02:42.047;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:02:59.794;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:03:00.959;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:03:04.383;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:03:07.670;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:03:07.720;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:03:12.823;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:03:22.481;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:03:27.108;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:03:44.852;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:03:45.025;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:03:49.437;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:03:52.738;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:03:52.786;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:03:57.866;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:04:07.534;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:04:11.215;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:04:12.165;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:04:29.922;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:04:30.094;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:04:34.505;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:04:37.808;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:04:37.847;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:04:42.923;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:04:52.592;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:04:57.224;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:05:14.980;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:05:15.153;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:05:19.565;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:05:22.878;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:05:22.896;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:05:27.976;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:05:37.658;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:05:42.286;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:05:59.043;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:06:00.219;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:06:04.631;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:06:07.950;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:06:07.951;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:06:12.029;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:06:22.721;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:06:27.362;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:06:44.112;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:06:45.287;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:06:49.700;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:06:52.013;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:06:53.016;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:06:57.103;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:07:07.786;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:07:12.418;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:07:29.174;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:07:30.357;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:07:34.768;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:07:37.065;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:07:37.085;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:07:42.163;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:07:52.851;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:07:57.493;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:08:14.229;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:08:15.419;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:08:19.830;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:08:22.128;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:08:22.147;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:08:27.216;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:08:37.919;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:08:42.553;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:08:59.272;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:09:00.490;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:09:04.901;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:09:07.192;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:09:07.222;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:09:11.065;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:09:12.271;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:09:22.981;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:09:27.612;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:09:44.339;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:09:45.560;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:09:49.973;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:09:52.255;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:09:52.287;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:09:57.334;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:10:07.041;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:10:12.677;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:10:29.403;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:10:30.629;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:10:34.040;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:10:37.318;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:10:37.357;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:10:42.388;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:10:52.094;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:10:57.739;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:11:14.457;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:11:15.697;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:11:19.110;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:11:22.380;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:11:22.415;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:11:27.442;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:11:37.150;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:11:42.797;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:11:59.511;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:12:00.767;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:12:04.176;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:12:07.443;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:12:07.483;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:12:12.500;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:12:22.203;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:12:27.869;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:12:44.576;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:12:45.836;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:12:49.240;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:12:52.491;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:12:52.553;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:12:57.562;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:13:07.264;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:13:12.929;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:13:29.637;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:13:30.904;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:13:34.311;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:13:37.550;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:13:37.623;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:13:42.617;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:13:52.329;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:13:57.987;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:14:11.161;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:14:14.702;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:14:15.973;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:14:19.382;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:14:22.608;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:14:22.692;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:14:27.678;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:14:37.381;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:14:42.046;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:14:59.761;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:15:00.037;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:15:04.438;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:15:07.672;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:15:07.760;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:15:12.736;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:15:22.445;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:15:27.105;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:15:44.826;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:15:45.100;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:15:49.502;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:15:52.736;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:15:52.829;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:15:57.790;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:16:07.537;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:16:12.165;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:16:29.873;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:16:30.170;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:16:34.567;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:16:37.799;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:16:37.887;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:16:42.837;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:16:52.609;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:16:57.224;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:17:14.940;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:17:15.238;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:17:19.609;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:17:22.855;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:17:22.956;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:17:27.887;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:17:37.669;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:17:42.270;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:17:59.000;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:18:00.307;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:18:04.679;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:18:07.020;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:18:07.916;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:18:12.936;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:18:22.752;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:18:27.330;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:18:44.066;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:18:45.375;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:18:49.747;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:18:52.088;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:18:52.981;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:18:57.986;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:19:07.816;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:19:11.255;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:19:12.390;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:19:29.127;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:19:30.427;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:19:34.816;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:19:37.041;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:19:37.158;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:19:42.042;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:19:52.909;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:19:57.449;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:20:14.195;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:20:15.491;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:20:19.874;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:20:22.102;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:20:22.228;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:20:27.098;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:20:37.970;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:20:42.519;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:20:59.247;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:21:00.546;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:21:04.941;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:21:07.167;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:21:07.290;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:21:12.155;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:21:22.034;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:21:27.581;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:21:44.309;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:21:45.615;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:21:50.010;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:21:52.233;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:21:52.336;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:21:57.207;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:22:07.091;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:22:12.634;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:22:29.376;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:22:30.686;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:22:34.079;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:22:37.286;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:22:37.405;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:22:42.264;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:22:52.151;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:22:57.694;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:23:14.454;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:23:15.756;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:23:19.147;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:23:22.349;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:23:22.467;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:23:27.320;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:23:37.215;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:23:42.756;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:23:59.511;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:24:00.825;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:24:04.216;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:24:07.414;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:24:07.539;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:24:11.102;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:24:12.379;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:24:22.278;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:24:27.834;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:24:44.576;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:24:45.894;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:24:49.285;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:24:52.469;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:24:52.609;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:24:57.436;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:25:07.342;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:25:12.926;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:25:29.639;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:25:30.954;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:25:34.350;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:25:37.528;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:25:37.677;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:25:42.494;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:25:52.406;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:25:57.985;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:26:14.691;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:26:15.022;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:26:19.419;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:26:22.588;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:26:22.731;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:26:27.547;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:26:37.467;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:26:42.045;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:26:59.740;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:27:00.085;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:27:04.481;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:27:07.651;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:27:07.798;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:27:12.605;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:27:22.536;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:27:27.106;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:27:44.790;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:27:45.144;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:27:49.550;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:27:52.714;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:27:52.871;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:27:57.661;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:28:07.601;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:28:12.164;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:28:29.851;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:28:30.213;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:28:34.619;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:28:37.779;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:28:37.940;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:28:42.719;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:28:52.654;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:28:57.225;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:29:11.198;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:29:14.916;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:29:15.280;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:29:19.688;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:29:22.843;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:29:23.011;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:29:27.767;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:29:37.701;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:29:42.285;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:29:59.980;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:30:00.343;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:30:04.748;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:30:07.080;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:30:07.918;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:30:12.817;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:30:22.750;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:30:27.338;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:30:44.045;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:30:45.403;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:30:49.813;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:30:52.150;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:30:52.987;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:30:57.861;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:31:07.826;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:31:12.398;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:31:29.102;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:31:30.458;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:31:34.880;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:31:37.049;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:31:37.207;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:31:42.922;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:31:52.892;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:31:57.456;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:32:14.184;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:32:15.524;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:32:19.946;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:32:22.110;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:32:22.277;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:32:27.976;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:32:37.951;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:32:42.507;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:32:59.241;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:33:00.593;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:33:05.016;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:33:07.177;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:33:07.345;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:33:12.033;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:33:23.012;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:33:27.567;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:33:44.300;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:33:45.662;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:33:50.081;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:33:52.241;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:33:52.405;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:33:57.091;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:34:08.079;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:34:11.041;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:34:12.649;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:34:29.364;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:34:30.731;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:34:35.131;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:34:37.301;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:34:37.466;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:34:42.148;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:34:53.130;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:34:57.729;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:35:14.445;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:35:15.795;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:35:20.196;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:35:22.358;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:35:22.535;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:35:27.196;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:35:38.194;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:35:42.791;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:35:59.496;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:36:00.861;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:36:05.261;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:36:07.427;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:36:07.590;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:36:12.253;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:36:23.256;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:36:27.854;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:36:44.550;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:36:45.925;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:36:50.329;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:36:52.491;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:36:52.657;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:36:57.307;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:37:08.322;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:37:12.908;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:37:29.615;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:37:30.984;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:37:35.398;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:37:37.547;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:37:37.724;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:37:42.372;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:37:53.386;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:37:57.969;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:38:14.667;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:38:15.051;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:38:20.475;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:38:22.609;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:38:22.794;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:38:27.429;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:38:38.440;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:38:42.027;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:38:59.714;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:39:00.118;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:39:05.544;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:39:07.669;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:39:07.855;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:39:11.132;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:39:12.485;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:39:23.500;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:39:27.109;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:39:44.777;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:39:45.185;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:39:50.613;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:39:52.733;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:39:52.916;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:39:57.546;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:40:08.562;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:40:12.172;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:40:29.841;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:40:30.252;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:40:35.682;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:40:37.798;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:40:37.986;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:40:42.607;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:40:53.627;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:40:57.235;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:41:14.944;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:41:15.322;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:41:20.751;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:41:22.043;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:41:22.858;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:41:27.661;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:41:38.697;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:41:42.299;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:41:59.000;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:42:00.388;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:42:05.826;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:42:07.111;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:42:07.913;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:42:12.718;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:42:23.748;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:42:27.359;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:42:44.065;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:42:45.456;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:42:50.895;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:42:52.182;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:42:52.969;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:42:57.778;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:43:08.803;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:43:12.437;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:43:29.148;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:43:30.524;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:43:35.960;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:43:37.025;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:43:37.261;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:43:42.836;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:43:53.867;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:43:57.495;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:44:11.224;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:44:14.213;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:44:15.593;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:44:21.030;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:44:22.090;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:44:22.335;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:44:27.887;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:44:38.923;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:44:42.556;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:44:59.271;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:45:00.657;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:45:06.090;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:45:07.154;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:45:07.407;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:45:12.933;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:45:23.983;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:45:27.626;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:45:44.336;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:45:45.721;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:45:51.156;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:45:52.219;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:45:52.478;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:45:57.989;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:46:08.039;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:46:12.694;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:46:29.399;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:46:30.779;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:46:36.242;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:46:37.284;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:46:37.563;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:46:42.040;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:46:53.103;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:46:57.767;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:47:14.461;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:47:15.846;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:47:21.311;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:47:22.347;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:47:22.627;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:47:27.097;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:47:38.164;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:47:42.837;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:47:59.520;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:48:00.914;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:48:06.377;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:48:07.399;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:48:07.694;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:48:12.163;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:48:23.221;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:48:27.898;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:48:44.583;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:48:45.983;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:48:51.476;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:48:52.459;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:48:52.761;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:48:57.217;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:49:08.283;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:49:11.071;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:49:12.958;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:49:29.650;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:49:30.051;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:49:36.541;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:49:37.514;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:49:37.831;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:49:42.275;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:49:53.344;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:49:57.036;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:50:14.711;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:50:15.110;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:50:21.607;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:50:22.576;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:50:22.899;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:50:27.335;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:50:38.411;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:50:42.096;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:50:59.761;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:51:00.175;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:51:06.699;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:51:07.640;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:51:07.966;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:51:12.392;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:51:23.473;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:51:27.158;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:51:44.824;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:51:45.242;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:51:51.772;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:51:52.026;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:51:52.705;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:51:57.446;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:52:08.533;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:52:12.218;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:52:29.881;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:52:30.311;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:52:36.871;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:52:37.102;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:52:37.765;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:52:42.497;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:52:53.596;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:52:57.279;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:53:14.937;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:53:15.379;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:53:21.933;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:53:22.174;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:53:22.823;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:53:27.547;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:53:38.651;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:53:42.338;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:53:59.002;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:54:00.442;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:54:06.005;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:54:07.248;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:54:07.879;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:54:11.169;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:54:12.603;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:54:23.716;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:54:27.397;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:54:44.065;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:54:45.508;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:54:51.074;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:54:52.323;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:54:52.939;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:54:57.659;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:55:08.773;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:55:12.452;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:55:29.129;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:55:30.575;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:55:36.141;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:55:37.402;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:55:37.001;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:55:42.717;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:55:53.838;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:55:57.507;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:56:14.191;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:56:15.635;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:56:21.216;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:56:22.064;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:56:22.466;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:56:27.766;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:56:38.905;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:56:42.556;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:56:59.249;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:57:00.700;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:57:06.285;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:57:07.119;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:57:07.535;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:57:12.823;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:57:23.965;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:57:27.616;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:57:44.311;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:57:45.771;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:57:51.361;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:57:52.179;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:57:52.609;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:57:57.871;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:58:08.021;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:58:12.672;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:58:29.371;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:58:30.823;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:58:36.431;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:58:37.236;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:58:37.676;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:58:42.929;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:58:53.092;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:58:57.732;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:59:11.017;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 06:59:14.430;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 06:59:15.891;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 06:59:21.505;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 06:59:22.300;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 06:59:22.739;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 06:59:27.985;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 06:59:38.154;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 06:59:42.793;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 06:59:59.480;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:00:00.954;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:00:06.585;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:00:07.356;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:00:07.808;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:00:12.040;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:00:23.221;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:00:27.855;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:00:44.540;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:00:45.012;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:00:51.654;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:00:52.423;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:00:52.877;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:00:57.115;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:01:08.282;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:01:12.940;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:01:29.605;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:01:30.069;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:01:36.736;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:01:37.488;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:01:37.951;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:01:42.169;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:01:53.344;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:01:57.994;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:02:14.669;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:02:15.140;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:02:21.826;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:02:22.547;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:02:23.022;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:02:27.226;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:02:38.394;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:02:42.073;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:02:59.731;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:03:00.211;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:03:06.897;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:03:07.602;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:03:08.092;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:03:12.285;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:03:23.463;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:03:27.133;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:03:44.792;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:03:45.278;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:03:51.977;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:03:52.667;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:03:53.163;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:03:57.342;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:04:08.532;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:04:11.114;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:04:12.192;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:04:29.851;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:04:30.346;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:04:36.043;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:04:37.727;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:04:38.233;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:04:42.400;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:04:53.590;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:04:57.252;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:05:14.915;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:05:15.410;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:05:21.108;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:05:22.788;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:05:23.305;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:05:27.461;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:05:38.655;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:05:42.311;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:05:59.976;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:06:00.477;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:06:06.176;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:06:07.853;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:06:08.361;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:06:12.515;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:06:23.721;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:06:27.400;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:06:44.029;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:06:45.545;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:06:51.241;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:06:52.910;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:06:53.429;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:06:57.592;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:07:08.785;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:07:12.465;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:07:29.082;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:07:30.605;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:07:36.312;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:07:37.972;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:07:38.499;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:07:42.649;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:07:53.850;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:07:57.506;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:08:14.135;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:08:15.675;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:08:21.385;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:08:22.029;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:08:23.568;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:08:27.704;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:08:38.918;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:08:42.564;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:08:59.199;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:09:00.742;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:09:06.492;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:09:07.094;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:09:08.655;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:09:11.211;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:09:12.760;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:09:23.983;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:09:27.619;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:09:44.262;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:09:45.816;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:09:51.565;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:09:52.158;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:09:53.727;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:09:57.819;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:10:08.041;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:10:12.684;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:10:29.327;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:10:30.885;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:10:36.644;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:10:37.222;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:10:38.801;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:10:42.868;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:10:53.106;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:10:57.749;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:11:14.386;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:11:15.947;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:11:21.710;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:11:22.281;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:11:23.855;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:11:27.908;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:11:38.157;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:11:42.827;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:11:59.447;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:12:00.025;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:12:06.780;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:12:07.345;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:12:08.930;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:12:12.956;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:12:23.204;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:12:27.886;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:12:44.511;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:12:45.104;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:12:51.853;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:12:52.399;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:12:53.002;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:12:58.009;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:13:08.266;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:13:12.947;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:13:29.572;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:13:30.166;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:13:36.928;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:13:37.460;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:13:38.095;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:13:43.065;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:13:53.332;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:13:57.001;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:14:11.059;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:14:14.636;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:14:15.235;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:14:21.999;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:14:22.517;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:14:23.166;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:14:28.122;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:14:38.393;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:14:42.060;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:14:59.711;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:15:00.333;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:15:06.059;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:15:07.580;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:15:08.242;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:15:13.178;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:15:23.456;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:15:27.087;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:15:44.776;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:15:45.398;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:15:51.125;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:15:52.642;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:15:53.321;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:15:58.234;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:16:08.521;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:16:12.146;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:16:29.842;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:16:30.462;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:16:36.192;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:16:37.706;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:16:38.389;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:16:43.286;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:16:53.586;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:16:57.207;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:17:14.947;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:17:15.531;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:17:21.261;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:17:22.765;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:17:23.468;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:17:28.342;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:17:38.652;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:17:42.268;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:17:59.017;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:18:00.600;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:18:06.317;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:18:07.827;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:18:08.541;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:18:13.398;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:18:23.706;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:18:27.330;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:18:44.100;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:18:45.675;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:18:51.384;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:18:52.889;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:18:53.619;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:18:58.446;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:19:08.770;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:19:11.156;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:19:12.397;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:19:29.165;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:19:30.737;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:19:36.455;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:19:37.951;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:19:38.689;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:19:43.497;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:19:53.831;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:19:57.458;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:20:14.231;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:20:15.802;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:20:21.511;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:20:22.015;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:20:23.762;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:20:28.547;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:20:38.897;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:20:42.516;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:20:59.300;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:21:00.867;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:21:06.577;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:21:07.087;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:21:08.823;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:21:13.603;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:21:23.956;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:21:27.612;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:21:44.366;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:21:45.935;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:21:51.646;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:21:52.152;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:21:53.891;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:21:58.659;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:22:08.021;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:22:12.667;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:22:29.436;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:22:31.018;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:22:36.714;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:22:37.209;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:22:38.949;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:22:43.712;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:22:53.072;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:22:57.726;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:23:14.488;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:23:15.091;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:23:21.784;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:23:22.272;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:23:24.027;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:23:28.776;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:23:38.137;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:23:42.787;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:23:59.550;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:24:00.156;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:24:06.854;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:24:07.336;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:24:08.099;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:24:11.251;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:24:13.834;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:24:23.201;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:24:27.854;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:24:44.614;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:24:45.260;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:24:51.925;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:24:52.404;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:24:53.169;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:24:58.883;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:25:08.262;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:25:12.951;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:25:29.679;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:25:30.334;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:25:36.996;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:25:37.467;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:25:38.248;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:25:43.937;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:25:53.326;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:25:57.003;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:26:14.726;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:26:15.405;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:26:21.063;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:26:22.529;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:26:23.316;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:26:28.009;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:26:38.391;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:26:42.056;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:26:59.787;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:27:00.459;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:27:06.130;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:27:07.592;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:27:08.389;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:27:13.072;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:27:23.464;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:27:27.119;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:27:44.850;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:27:45.534;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:27:51.195;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:27:52.692;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:27:53.462;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:27:58.120;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:28:08.521;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:28:12.178;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:28:29.914;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:28:30.598;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:28:36.264;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:28:37.760;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:28:38.528;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:28:43.170;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:28:53.585;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:28:57.238;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:29:11.092;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:29:14.980;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:29:15.676;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:29:21.334;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:29:22.818;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:29:23.598;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:29:28.229;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:29:38.659;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:29:42.316;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:29:59.043;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:30:00.741;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:30:06.390;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:30:07.876;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:30:08.666;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:30:13.286;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:30:23.720;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:30:27.376;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:30:44.103;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:30:45.849;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:30:51.455;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:30:52.929;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:30:53.735;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:30:58.362;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:31:08.784;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:31:12.461;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:31:29.161;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:31:30.943;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:31:36.520;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:31:37.996;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:31:38.810;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:31:43.444;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:31:53.850;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:31:57.527;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:32:14.220;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:32:15.005;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:32:21.580;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:32:22.061;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:32:23.878;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:32:28.505;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:32:38.913;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:32:42.611;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:32:59.281;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:33:00.073;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:33:06.650;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:33:07.129;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:33:08.945;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:33:13.564;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:33:23.964;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:33:27.688;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:33:44.341;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:33:45.144;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:33:51.726;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:33:52.195;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:33:54.012;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:33:58.622;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:34:08.021;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:34:11.185;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:34:12.753;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:34:29.399;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:34:30.215;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:34:36.794;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:34:37.256;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:34:38.068;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:34:43.688;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:34:53.079;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:34:57.842;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:35:14.460;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:35:15.286;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:35:21.862;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:35:22.318;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:35:23.139;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:35:28.764;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:35:38.141;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:35:42.908;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:35:59.516;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:36:00.353;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:36:06.939;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:36:07.380;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:36:08.205;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:36:13.818;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:36:23.208;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:36:27.975;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:36:44.577;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:36:45.414;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:36:52.012;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:36:52.444;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:36:53.275;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:36:58.872;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:37:08.265;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:37:12.034;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:37:29.643;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:37:30.490;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:37:36.072;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:37:37.495;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:37:38.344;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:37:43.931;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:37:53.328;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:37:57.094;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:38:14.695;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:38:15.576;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:38:21.142;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:38:22.555;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:38:23.410;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:38:28.992;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:38:38.393;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:38:42.170;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:38:59.749;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:39:00.650;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:39:06.212;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:39:07.620;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:39:08.476;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:39:11.030;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:39:13.050;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:39:23.454;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:39:27.236;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:39:44.813;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:39:45.721;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:39:51.283;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:39:52.685;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:39:53.547;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:39:58.107;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:40:08.523;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:40:12.306;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:40:29.861;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:40:30.791;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:40:36.354;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:40:37.750;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:40:38.620;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:40:43.165;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:40:53.589;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:40:57.382;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:41:14.916;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:41:15.859;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:41:21.424;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:41:22.808;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:41:23.690;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:41:28.236;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:41:38.653;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:41:42.478;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:41:59.973;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:42:00.928;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:42:06.491;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:42:07.869;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:42:08.759;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:42:13.308;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:42:23.712;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:42:27.533;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:42:44.041;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:42:45.998;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:42:51.554;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:42:52.931;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:42:53.829;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:42:58.366;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:43:08.785;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:43:12.592;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:43:29.102;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:43:30.065;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:43:36.594;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:43:37.996;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:43:38.893;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:43:43.419;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:43:53.854;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:43:57.658;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:44:11.125;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:44:14.165;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:44:15.158;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:44:21.664;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:44:22.059;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:44:23.962;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:44:28.477;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:44:38.902;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:44:42.724;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:44:59.230;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:45:00.223;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:45:06.729;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:45:07.119;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:45:08.031;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:45:13.547;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:45:23.951;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:45:27.797;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:45:44.290;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:45:45.288;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:45:51.799;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:45:52.184;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:45:53.100;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:45:58.604;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:46:08.018;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:46:12.857;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:46:29.358;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:46:30.360;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:46:36.868;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:46:37.249;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:46:38.168;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:46:43.708;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:46:53.081;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:46:57.919;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:47:14.419;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:47:15.421;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:47:21.937;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:47:22.310;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:47:23.228;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:47:28.772;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:47:38.142;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:47:42.010;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:47:59.474;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:48:00.492;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:48:06.011;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:48:07.364;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:48:08.293;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:48:13.832;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:48:23.203;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:48:27.070;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:48:44.533;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:48:45.560;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:48:51.083;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:48:52.429;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:48:53.360;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:48:58.883;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:49:08.266;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:49:11.218;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:49:12.126;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:49:29.598;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:49:30.629;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:49:36.153;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:49:37.488;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:49:38.431;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:49:43.935;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:49:53.327;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:49:57.190;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:50:14.662;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:50:15.699;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:50:21.208;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:50:22.550;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:50:23.501;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:50:28.996;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:50:38.393;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:50:42.280;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:50:59.720;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:51:00.767;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:51:06.276;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:51:07.609;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:51:08.561;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:51:13.075;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:51:23.454;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:51:27.344;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:51:44.780;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:51:45.827;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:51:51.347;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:51:52.673;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:51:53.628;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:51:58.128;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:52:08.515;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:52:12.399;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:52:29.843;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:52:30.897;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:52:36.419;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:52:37.725;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:52:38.697;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:52:43.202;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:52:53.572;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:52:57.459;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:53:14.899;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:53:15.970;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:53:21.489;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:53:22.787;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:53:23.763;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:53:28.256;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:53:38.637;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:53:42.535;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:53:59.966;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:54:00.037;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:54:06.561;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:54:07.852;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:54:08.840;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:54:11.063;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:54:13.310;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:54:23.702;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:54:27.596;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:54:44.030;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:54:45.106;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:54:51.627;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:54:52.908;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:54:53.910;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:54:58.366;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:55:08.762;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:55:12.656;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:55:29.090;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:55:30.170;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:55:36.692;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:55:37.970;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:55:38.981;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:55:43.423;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:55:53.821;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:55:57.716;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:56:14.151;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:56:15.233;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:56:21.736;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:56:22.033;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:56:23.044;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:56:28.500;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:56:38.876;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:56:42.778;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:56:59.217;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:57:00.301;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:57:06.806;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:57:07.098;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:57:08.114;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:57:13.557;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:57:23.929;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:57:27.837;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:57:44.283;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:57:45.372;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:57:51.878;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:57:52.161;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:57:53.185;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:57:58.613;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:58:08.982;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:58:12.896;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:58:29.352;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:58:30.442;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:58:36.952;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:58:37.226;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:58:38.254;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:58:43.673;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:58:53.044;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:58:57.956;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:59:11.162;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 07:59:14.423;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 07:59:15.507;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 07:59:22.026;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 07:59:22.291;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 07:59:23.324;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 07:59:28.732;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 07:59:38.101;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 07:59:42.055;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 07:59:59.483;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:00:00.574;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:00:07.097;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:00:07.352;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:00:08.386;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:00:13.790;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:00:23.163;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:00:27.114;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:00:44.530;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:00:45.639;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:00:52.170;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:00:52.411;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:00:53.457;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:00:58.846;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:01:08.223;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:01:12.167;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:01:29.593;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:01:30.699;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:01:37.238;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:01:37.476;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:01:38.516;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:01:43.919;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:01:53.282;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:01:57.225;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:02:14.661;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:02:15.768;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:02:22.312;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:02:22.538;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:02:23.586;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:02:28.965;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:02:38.342;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:02:42.280;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:02:59.729;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:03:00.836;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:03:07.386;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:03:07.596;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:03:08.653;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:03:13.028;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:03:23.404;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:03:27.339;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:03:44.792;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:03:45.906;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:03:52.453;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:03:52.650;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:03:53.724;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:03:58.092;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:04:08.462;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:04:11.255;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:04:12.394;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:04:29.848;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:04:30.976;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:04:37.522;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:04:37.705;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:04:38.793;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:04:43.161;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:04:53.520;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:04:57.464;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:05:14.908;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:05:15.033;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:05:22.586;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:05:22.769;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:05:23.864;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:05:28.216;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:05:38.584;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:05:42.540;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:05:59.969;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:06:00.102;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:06:07.653;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:06:07.829;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:06:08.921;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:06:13.284;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:06:23.645;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:06:27.600;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:06:44.029;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:06:45.166;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:06:52.723;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:06:52.902;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:06:53.989;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:06:58.371;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:07:08.695;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:07:12.666;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:07:29.099;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:07:30.248;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:07:37.799;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:07:37.981;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:07:38.061;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:07:43.427;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:07:53.751;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:07:57.726;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:08:14.163;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:08:15.321;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:08:22.043;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:08:22.883;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:08:23.134;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:08:28.483;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:08:38.816;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:08:42.832;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:08:59.227;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:09:00.384;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:09:07.104;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:09:07.951;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:09:08.203;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:09:11.099;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:09:13.535;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:09:23.881;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:09:27.928;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:09:44.288;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:09:45.454;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:09:52.028;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:09:52.169;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:09:53.276;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:09:58.586;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:10:08.942;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:10:12.993;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:10:29.354;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:10:30.525;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:10:37.104;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:10:37.232;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:10:38.347;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:10:43.651;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:10:54.005;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:10:57.056;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:11:14.419;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:11:15.604;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:11:22.171;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:11:22.296;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:11:23.415;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:11:28.732;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:11:38.069;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:11:42.117;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:11:59.477;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:12:00.667;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:12:07.232;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:12:07.359;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:12:08.497;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:12:13.794;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:12:23.131;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:12:27.173;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:12:44.540;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:12:45.736;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:12:52.303;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:12:52.429;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:12:53.570;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:12:58.851;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:13:08.192;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:13:12.233;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:13:29.599;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:13:30.806;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:13:37.380;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:13:37.490;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:13:38.640;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:13:43.909;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:13:53.274;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:13:57.301;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:14:11.189;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:14:14.665;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:14:15.876;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:14:22.454;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:14:22.562;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:14:23.705;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:14:28.985;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:14:38.331;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:14:42.372;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:14:59.727;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:15:00.941;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:15:07.511;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:15:07.614;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:15:08.777;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:15:13.045;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:15:23.386;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:15:27.424;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:15:44.788;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:15:46.008;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:15:52.576;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:15:52.670;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:15:53.846;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:15:58.106;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:16:08.458;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:16:12.475;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:16:29.847;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:16:30.069;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:16:37.645;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:16:37.742;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:16:38.915;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:16:43.190;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:16:53.520;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:16:57.544;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:17:14.908;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:17:15.137;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:17:22.713;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:17:22.806;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:17:23.986;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:17:28.247;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:17:38.581;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:17:42.649;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:17:59.973;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:18:00.207;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:18:07.784;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:18:07.870;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:18:08.053;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:18:13.313;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:18:23.631;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:18:27.710;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:18:44.029;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:18:45.278;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:18:52.855;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:18:52.941;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:18:53.114;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:18:58.374;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:19:08.685;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:19:11.036;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:19:12.771;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:19:29.093;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:19:30.347;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:19:37.927;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:19:37.003;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:19:38.184;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:19:43.434;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:19:53.749;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:19:57.833;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:20:14.159;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:20:15.412;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:20:22.071;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:20:22.989;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:20:23.255;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:20:28.492;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:20:38.814;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:20:42.910;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:20:59.220;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:21:00.479;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:21:07.055;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:21:07.128;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:21:08.320;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:21:13.554;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:21:23.887;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:21:27.986;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:21:44.283;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:21:45.548;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:21:52.116;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:21:52.192;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:21:53.387;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:21:58.625;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:22:08.956;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:22:12.047;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:22:29.349;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:22:30.617;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:22:37.186;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:22:37.246;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:22:38.460;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:22:43.679;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:22:53.022;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:22:57.105;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:23:14.404;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:23:15.687;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:23:22.246;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:23:22.301;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:23:23.525;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:23:28.741;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:23:38.086;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:23:42.165;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:23:59.465;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:24:00.758;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:24:07.318;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:24:07.358;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:24:08.588;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:24:11.129;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:24:13.795;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:24:23.150;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:24:27.226;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:24:44.530;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:24:45.829;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:24:52.391;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:24:52.418;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:24:53.656;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:24:58.851;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:25:08.212;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:25:12.290;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:25:29.593;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:25:30.897;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:25:37.461;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:25:37.476;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:25:38.726;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:25:43.914;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:25:53.273;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:25:57.359;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:26:14.646;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:26:15.968;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:26:22.521;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:26:22.542;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:26:23.794;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:26:28.986;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:26:38.346;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:26:42.416;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:26:59.708;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:27:00.035;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:27:07.588;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:27:07.602;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:27:08.855;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:27:13.045;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:27:23.408;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:27:27.476;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:27:44.773;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:27:45.119;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:27:52.657;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:27:52.662;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:27:53.927;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:27:58.117;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:28:08.481;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:28:12.533;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:28:29.833;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:28:30.189;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:28:37.728;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:28:37.728;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:28:38.997;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:28:43.177;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:28:53.535;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:28:57.595;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:29:11.222;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:29:14.898;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:29:15.259;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:29:22.793;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:29:22.803;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:29:23.066;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:29:28.233;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:29:38.589;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:29:42.667;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:29:59.963;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:30:00.325;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:30:07.861;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:30:07.864;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:30:08.136;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:30:13.291;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:30:23.655;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:30:27.727;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:30:44.026;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:30:45.392;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:30:52.919;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:30:52.923;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:30:53.205;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:30:58.346;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:31:08.714;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:31:12.815;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:31:29.090;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:31:30.452;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:31:37.983;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:31:38.018;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:31:38.276;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:31:43.409;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:31:53.773;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:31:57.876;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:32:14.157;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:32:15.531;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:32:22.047;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:32:22.086;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:32:23.347;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:32:28.477;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:32:38.854;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:32:42.937;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:32:59.217;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:33:00.626;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:33:07.103;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:33:07.146;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:33:08.414;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:33:13.535;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:33:23.921;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:33:27.999;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:33:44.279;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:33:45.696;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:33:52.169;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:33:52.216;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:33:53.480;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:33:58.608;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:34:08.984;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:34:11.064;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:34:12.058;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:34:29.342;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:34:30.767;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:34:37.230;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:34:37.285;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:34:38.550;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:34:43.670;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:34:53.042;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:34:57.118;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:35:14.409;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:35:15.838;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:35:22.298;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:35:22.352;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:35:23.618;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:35:28.728;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:35:38.109;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:35:42.179;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:35:59.463;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:36:00.946;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:36:07.353;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:36:07.425;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:36:08.682;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:36:13.786;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:36:23.177;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:36:27.234;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:36:44.518;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:36:45.004;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:36:52.419;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:36:52.506;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:36:53.749;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:36:58.852;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:37:08.244;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:37:12.296;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:37:29.580;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:37:30.093;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:37:37.482;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:37:37.577;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:37:38.818;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:37:43.915;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:37:53.305;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:37:57.353;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:38:14.636;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:38:15.155;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:38:22.540;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:38:22.647;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:38:23.887;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:38:28.971;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:38:38.370;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:38:42.404;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:38:59.712;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:39:00.225;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:39:07.602;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:39:07.718;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:39:08.957;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:39:11.156;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:39:13.025;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:39:23.434;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:39:27.465;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:39:44.777;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:39:45.298;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:39:52.665;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:39:52.789;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:39:53.027;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:39:58.074;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:40:08.492;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:40:12.525;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:40:29.842;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:40:30.379;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:40:37.729;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:40:37.852;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:40:38.096;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:40:43.126;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:40:53.551;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:40:57.588;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:41:14.903;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:41:15.445;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:41:22.793;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:41:22.921;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:41:23.155;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:41:28.195;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:41:38.615;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:41:42.650;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:41:59.966;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:42:00.520;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:42:07.853;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:42:07.990;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:42:08.224;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:42:13.253;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:42:23.668;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:42:27.704;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:42:44.030;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:42:45.589;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:42:52.062;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:42:52.910;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:42:53.294;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:42:58.312;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:43:08.705;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:43:12.763;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:43:29.094;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:43:30.677;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:43:37.130;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:43:37.975;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:43:38.363;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:43:43.387;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:43:53.764;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:43:57.851;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:44:11.246;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:44:14.159;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:44:15.752;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:44:22.039;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:44:22.195;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:44:23.435;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:44:28.467;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:44:38.819;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:44:42.904;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:44:59.218;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:45:00.818;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:45:07.103;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:45:07.257;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:45:08.504;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:45:13.525;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:45:23.884;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:45:27.963;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:45:44.279;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:45:45.883;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:45:52.160;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:45:52.293;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:45:53.575;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:45:58.593;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:46:08.944;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:46:13.023;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:46:29.344;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:46:30.948;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:46:37.218;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:46:37.363;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:46:38.642;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:46:43.658;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:46:54.009;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:46:58.077;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:47:14.409;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:47:16.012;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:47:22.280;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:47:22.435;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:47:23.711;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:47:28.731;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:47:39.073;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:47:43.138;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:47:59.467;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:48:00.071;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:48:07.334;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:48:07.506;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:48:08.776;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:48:13.794;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:48:24.139;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:48:28.197;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:48:44.538;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:48:45.161;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:48:52.391;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:48:52.572;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:48:53.834;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:48:58.859;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:49:09.203;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:49:11.095;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:49:13.256;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:49:29.612;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:49:30.231;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:49:37.451;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:49:37.642;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:49:38.907;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:49:43.914;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:49:54.254;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:49:58.315;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:50:14.679;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:50:15.296;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:50:22.513;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:50:22.705;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:50:23.977;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:50:28.966;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:50:39.318;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:50:43.375;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:50:59.744;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:51:00.374;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:51:07.570;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:51:07.771;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:51:08.039;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:51:13.015;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:51:24.382;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:51:28.435;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:51:44.803;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:51:45.434;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:51:52.630;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:51:52.842;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:51:53.107;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:51:58.063;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:52:09.441;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:52:13.492;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:52:29.860;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:52:30.504;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:52:37.712;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:52:37.911;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:52:38.167;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:52:43.114;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:52:54.503;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:52:58.553;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:53:14.914;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:53:15.575;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:53:22.768;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:53:22.981;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:53:23.236;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:53:28.173;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:53:39.551;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:53:43.612;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:53:59.973;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:54:00.646;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:54:07.054;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:54:07.833;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:54:08.314;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:54:11.186;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:54:13.231;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:54:24.616;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:54:28.671;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:54:44.037;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:54:45.717;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:54:52.115;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:54:52.891;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:54:53.384;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:54:58.288;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:55:09.671;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:55:13.741;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:55:29.103;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:55:30.784;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:55:37.184;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:55:37.952;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:55:38.453;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:55:43.354;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:55:54.731;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:55:58.801;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:56:14.168;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:56:15.855;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:56:22.016;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:56:22.249;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:56:23.523;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:56:28.427;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:56:39.796;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:56:43.855;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:56:59.227;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:57:00.917;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:57:07.078;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:57:07.315;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:57:08.608;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:57:13.485;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:57:24.860;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:57:28.915;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:57:44.293;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:57:45.027;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:57:52.136;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:57:52.389;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:57:53.679;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:57:58.534;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:58:09.922;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:58:13.975;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:58:29.358;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:58:30.088;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:58:37.200;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:58:37.459;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:58:38.750;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:58:43.591;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:58:54.982;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:58:58.034;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:59:11.030;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 08:59:14.426;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 08:59:15.149;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 08:59:22.265;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 08:59:22.516;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 08:59:23.822;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 08:59:28.648;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 08:59:39.040;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 08:59:43.086;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 08:59:59.489;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:00:00.213;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:00:07.329;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:00:07.581;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:00:08.892;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:00:13.703;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:00:24.101;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:00:28.143;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:00:44.551;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:00:45.290;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:00:52.390;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:00:52.646;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:00:53.963;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:00:58.756;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:01:09.163;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:01:13.203;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:01:29.615;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:01:30.360;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:01:37.465;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:01:37.701;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:01:38.040;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:01:43.819;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:01:54.213;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:01:58.259;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:02:14.680;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:02:15.418;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:02:22.526;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:02:22.768;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:02:23.110;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:02:28.869;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:02:39.261;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:02:43.319;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:02:59.741;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:03:00.486;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:03:07.583;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:03:07.835;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:03:08.176;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:03:13.927;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:03:24.324;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:03:28.379;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:03:44.802;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:03:45.552;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:03:52.640;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:03:52.904;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:03:53.242;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:03:58.983;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:04:09.382;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:04:11.127;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:04:13.425;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:04:29.859;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:04:30.622;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:04:37.700;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:04:37.975;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:04:38.321;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:04:43.039;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:04:54.441;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:04:58.479;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:05:14.924;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:05:15.687;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:05:22.041;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:05:22.760;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:05:23.390;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:05:28.096;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:05:39.503;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:05:43.533;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:05:59.986;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:06:00.754;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:06:07.110;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:06:07.826;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:06:08.455;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:06:13.157;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:06:24.561;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:06:28.595;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:06:44.047;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:06:45.816;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:06:52.179;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:06:52.888;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:06:53.515;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:06:58.205;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:07:09.622;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:07:13.656;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:07:29.120;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:07:30.886;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:07:37.241;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:07:37.945;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:07:38.585;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:07:43.252;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:07:54.686;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:07:58.717;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:08:14.176;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:08:15.966;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:08:22.309;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:08:23.007;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:08:23.647;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:08:28.303;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:08:39.742;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:08:43.778;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:08:59.238;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:09:00.050;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:09:07.071;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:09:07.379;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:09:08.722;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:09:11.224;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:09:13.362;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:09:24.807;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:09:28.837;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:09:44.297;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:09:45.117;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:09:52.129;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:09:52.454;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:09:53.792;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:09:58.422;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:10:09.862;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:10:13.920;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:10:29.361;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:10:30.177;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:10:37.194;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:10:37.515;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:10:38.859;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:10:43.478;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:10:54.921;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:10:59.016;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:11:14.420;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:11:15.253;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:11:22.258;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:11:22.581;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:11:23.915;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:11:28.539;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:11:39.983;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:11:44.068;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:11:59.477;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:12:00.316;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:12:07.322;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:12:07.649;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:12:08.986;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:12:13.594;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:12:24.041;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:12:29.128;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:12:44.545;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:12:45.387;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:12:52.380;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:12:52.721;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:12:54.017;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:12:58.651;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:13:09.102;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:13:14.187;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:13:29.612;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:13:30.447;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:13:37.444;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:13:37.792;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:13:38.086;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:13:43.704;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:13:54.166;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:13:59.247;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:14:11.068;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:14:14.682;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:14:15.557;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:14:22.509;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:14:22.858;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:14:23.156;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:14:28.753;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:14:39.211;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:14:44.310;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:14:59.740;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:15:00.624;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:15:07.573;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:15:07.924;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:15:08.225;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:15:13.808;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:15:24.277;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:15:29.369;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:15:44.805;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:15:45.684;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:15:52.630;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:15:52.990;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:15:53.296;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:15:58.858;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:16:09.348;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:16:14.425;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:16:29.864;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:16:30.744;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:16:37.056;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:16:37.694;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:16:38.362;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:16:43.914;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:16:54.434;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:16:59.488;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:17:14.930;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:17:15.810;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:17:22.126;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:17:22.754;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:17:23.436;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:17:28.970;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:17:39.493;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:17:44.543;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:17:59.993;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:18:00.919;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:18:07.196;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:18:07.816;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:18:08.493;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:18:13.027;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:18:24.553;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:18:29.605;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:18:44.061;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:18:45.985;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:18:52.265;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:18:52.876;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:18:53.559;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:18:58.083;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:19:09.616;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:19:11.164;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:19:14.667;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:19:29.119;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:19:30.045;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:19:37.335;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:19:37.925;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:19:38.628;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:19:43.138;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:19:54.681;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:19:59.749;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:20:14.187;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:20:15.112;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:20:22.403;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:20:22.976;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:20:23.698;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:20:28.194;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:20:39.742;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:20:44.808;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:20:59.256;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:21:00.181;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:21:07.041;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:21:07.473;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:21:08.764;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:21:13.251;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:21:24.808;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:21:29.871;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:21:44.325;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:21:45.251;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:21:52.104;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:21:52.541;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:21:53.833;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:21:58.333;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:22:09.870;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:22:14.934;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:22:29.392;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:22:30.352;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:22:37.157;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:22:37.603;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:22:38.894;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:22:43.394;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:22:54.932;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:22:59.993;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:23:14.448;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:23:15.418;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:23:22.221;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:23:22.674;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:23:23.961;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:23:28.445;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:23:39.990;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:23:44.051;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:23:59.500;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:24:00.487;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:24:07.285;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:24:07.746;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:24:08.030;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:24:11.011;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:24:13.494;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:24:24.056;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:24:29.111;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:24:44.565;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:24:45.557;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:24:52.348;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:24:52.813;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:24:53.100;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:24:58.554;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:25:09.111;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:25:14.171;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:25:29.627;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:25:30.626;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:25:37.411;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:25:37.882;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:25:38.161;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:25:43.618;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:25:54.176;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:25:59.230;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:26:14.686;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:26:15.697;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:26:22.475;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:26:22.951;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:26:23.228;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:26:28.674;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:26:39.231;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:26:44.287;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:26:59.740;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:27:00.760;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:27:07.539;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:27:08.018;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:27:08.298;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:27:13.733;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:27:24.289;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:27:29.348;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:27:44.797;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:27:45.830;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:27:52.089;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:27:52.599;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:27:53.367;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:27:58.790;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:28:09.350;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:28:14.429;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:28:29.849;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:28:30.901;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:28:37.154;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:28:37.663;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:28:38.435;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:28:43.848;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:28:54.415;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:28:59.508;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:29:11.107;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:29:14.913;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:29:15.970;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:29:22.214;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:29:22.729;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:29:23.505;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:29:28.909;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:29:39.462;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:29:44.567;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:29:59.988;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:30:00.032;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:30:07.278;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:30:07.788;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:30:08.568;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:30:13.969;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:30:24.504;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:30:29.622;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:30:44.052;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:30:45.098;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:30:52.354;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:30:52.847;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:30:53.638;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:30:58.015;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:31:09.562;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:31:14.683;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:31:29.117;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:31:30.159;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:31:37.423;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:31:37.909;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:31:38.709;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:31:43.084;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:31:54.626;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:31:59.732;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:32:14.183;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:32:15.227;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:32:22.492;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:32:22.969;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:32:23.804;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:32:28.137;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:32:39.687;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:32:44.791;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:32:59.240;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:33:00.296;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:33:07.021;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:33:07.562;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:33:08.864;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:33:13.197;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:33:24.742;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:33:29.846;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:33:44.300;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:33:45.367;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:33:52.080;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:33:52.641;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:33:53.931;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:33:58.254;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:34:09.804;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:34:11.203;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:34:14.906;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:34:29.389;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:34:30.444;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:34:37.137;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:34:37.729;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:34:39.009;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:34:43.318;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:34:54.861;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:34:59.966;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:35:14.455;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:35:15.510;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:35:22.198;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:35:22.798;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:35:23.079;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:35:28.377;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:35:39.921;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:35:44.025;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:35:59.508;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:36:00.578;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:36:07.260;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:36:07.847;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:36:08.140;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:36:13.437;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:36:24.983;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:36:29.076;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:36:44.570;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:36:45.645;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:36:52.323;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:36:52.935;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:36:53.206;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:36:58.516;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:37:09.040;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:37:14.133;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:37:29.652;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:37:30.713;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:37:37.374;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:37:37.005;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:37:38.256;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:37:43.576;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:37:54.105;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:37:59.192;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:38:14.709;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:38:15.780;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:38:22.074;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:38:22.435;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:38:23.325;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:38:28.648;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:38:39.167;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:38:44.253;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:38:59.792;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:39:00.860;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:39:07.156;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:39:07.499;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:39:08.397;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:39:11.053;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:39:13.705;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:39:24.221;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:39:29.313;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:39:44.851;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:39:45.931;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:39:52.248;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:39:52.559;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:39:53.478;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:39:58.761;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:40:09.281;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:40:14.373;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:40:29.919;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:40:30.002;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:40:37.316;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:40:37.618;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:40:38.538;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:40:43.815;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:40:54.345;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:40:59.434;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:41:14.975;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:41:15.068;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:41:22.389;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:41:22.682;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:41:23.608;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:41:28.859;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:41:39.443;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:41:44.489;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:41:59.041;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:42:00.127;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:42:07.457;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:42:07.738;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:42:08.695;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:42:13.917;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:42:24.493;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:42:29.543;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:42:44.106;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:42:45.197;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:42:52.527;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:42:52.799;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:42:53.771;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:42:58.966;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:43:09.542;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:43:14.603;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:43:29.170;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:43:30.259;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:43:37.628;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:43:37.858;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:43:38.841;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:43:43.022;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:43:54.606;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:43:59.663;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:44:11.157;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:44:14.228;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:44:15.329;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:44:22.727;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:44:22.920;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:44:23.922;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:44:28.080;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:44:39.661;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:44:44.722;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:44:59.295;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:45:00.395;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:45:07.800;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:45:07.983;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:45:08.993;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:45:13.137;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:45:24.721;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:45:29.787;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:45:44.351;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:45:45.443;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:45:52.064;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:45:52.888;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:45:53.070;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:45:58.194;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:46:09.812;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:46:14.884;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:46:29.408;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:46:30.512;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:46:37.119;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:46:37.958;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:46:38.129;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:46:43.256;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:46:54.876;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:46:59.943;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:47:14.465;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:47:15.579;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:47:22.042;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:47:22.188;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:47:23.204;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:47:28.318;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:47:39.941;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:47:45.018;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:47:59.531;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:48:00.649;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:48:07.116;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:48:07.264;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:48:08.268;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:48:13.376;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:48:24.011;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:48:29.079;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:48:44.592;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:48:45.719;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:48:52.181;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:48:52.330;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:48:53.336;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:48:58.433;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:49:10.070;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:49:11.250;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:49:14.139;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:49:29.657;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:49:30.788;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:49:37.250;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:49:37.387;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:49:38.407;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:49:43.492;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:49:55.131;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:49:59.196;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:50:14.720;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:50:15.854;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:50:22.311;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:50:22.449;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:50:23.466;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:50:28.545;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:50:40.197;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:50:44.256;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:50:59.783;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:51:00.921;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:51:07.387;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:51:07.512;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:51:08.536;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:51:13.603;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:51:25.288;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:51:29.325;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:51:44.843;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:51:45.992;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:51:52.510;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:51:52.568;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:51:53.613;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:51:58.657;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:52:10.353;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:52:14.384;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:52:29.907;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:52:30.061;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:52:37.580;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:52:37.625;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:52:38.682;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:52:43.714;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:52:55.422;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:52:59.433;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:53:14.964;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:53:15.130;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:53:22.649;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:53:22.687;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:53:23.756;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:53:28.771;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:53:40.482;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:53:44.493;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:53:59.026;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:54:00.201;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:54:07.716;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:54:07.749;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:54:08.831;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:54:11.097;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:54:13.828;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:54:25.536;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:54:29.555;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:54:44.092;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:54:45.270;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:54:52.789;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:54:52.808;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:54:53.904;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:54:58.879;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:55:10.592;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:55:14.646;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:55:29.155;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:55:30.337;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:55:37.860;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:55:37.868;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:55:38.974;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:55:43.933;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:55:55.656;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:55:59.713;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:56:14.216;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:56:15.404;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:56:22.917;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:56:22.931;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:56:23.032;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:56:28.004;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:56:40.721;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:56:44.780;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:56:59.270;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:57:00.472;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:57:07.992;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:57:07.995;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:57:08.103;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:57:13.063;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:57:25.776;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:57:29.835;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:57:44.332;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:57:45.537;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:57:52.051;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:57:52.060;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:57:53.169;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:57:58.114;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:58:10.844;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:58:14.894;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:58:29.389;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:58:30.608;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:58:37.108;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:58:37.125;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:58:38.228;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:58:43.169;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:58:55.908;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:58:59.943;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:59:11.193;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 09:59:14.452;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 09:59:15.677;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 09:59:22.173;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 09:59:22.191;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 09:59:23.300;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 09:59:28.217;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 09:59:40.961;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 09:59:44.003;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 09:59:59.513;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:00:00.745;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:00:07.245;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:00:07.249;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:00:08.369;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:00:13.277;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:00:25.024;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:00:29.062;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:00:44.570;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:00:45.811;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:00:52.308;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:00:52.316;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:00:53.441;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:00:58.334;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:01:10.082;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:01:14.139;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:01:29.636;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:01:30.876;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:01:37.369;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:01:37.385;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:01:38.508;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:01:43.396;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:01:55.142;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:01:59.195;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:02:14.703;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:02:15.944;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:02:22.430;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:02:22.454;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:02:23.557;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:02:28.444;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:02:40.218;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:02:44.246;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:02:59.763;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:03:01.017;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:03:07.484;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:03:07.526;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:03:08.623;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:03:13.504;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:03:25.281;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:03:29.307;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:03:44.824;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:03:46.088;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:03:52.548;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:03:52.602;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:03:53.688;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:03:58.567;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:04:10.346;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:04:11.040;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:04:14.369;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:04:29.882;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:04:31.153;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:04:37.604;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:04:37.666;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:04:38.759;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:04:43.628;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:04:55.406;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:04:59.431;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:05:14.947;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:05:16.219;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:05:22.664;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:05:22.731;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:05:23.829;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:05:28.673;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:05:40.483;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:05:44.533;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:05:59.008;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:06:01.285;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:06:07.737;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:06:07.807;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:06:08.920;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:06:13.731;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:06:25.534;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:06:29.585;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:06:45.068;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:06:46.344;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:06:52.798;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:06:52.877;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:06:53.988;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:06:58.792;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:07:10.591;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:07:14.646;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:07:30.128;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:07:31.412;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:07:37.880;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:07:37.948;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:07:38.052;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:07:43.848;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:07:55.653;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:07:59.706;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:08:15.186;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:08:16.471;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:08:22.935;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:08:23.019;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:08:23.123;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:08:28.905;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:08:40.721;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:08:44.810;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:09:00.249;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:09:01.541;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:09:07.094;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:09:07.001;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:09:08.185;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:09:11.135;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:09:13.952;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:09:25.781;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:09:29.873;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:09:45.309;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:09:46.612;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:09:52.063;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:09:52.164;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:09:53.260;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:09:58.013;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:10:10.842;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:10:14.935;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:10:30.374;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:10:31.681;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:10:37.120;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:10:37.233;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:10:38.330;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:10:43.070;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:10:55.909;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:10:59.995;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:11:15.435;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:11:16.748;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:11:22.185;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:11:22.303;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:11:23.397;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:11:28.142;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:11:40.971;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:11:44.061;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:12:00.479;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:12:01.816;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:12:07.250;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:12:07.369;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:12:08.477;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:12:13.204;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:12:25.035;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:12:29.119;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:12:45.545;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:12:46.884;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:12:52.311;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:12:52.436;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:12:53.545;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:12:58.262;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:13:10.099;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:13:14.182;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:13:30.609;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:13:31.953;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:13:37.369;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:13:37.507;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:13:38.614;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:13:43.319;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:13:55.160;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:13:59.247;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:14:11.228;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:14:15.677;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:14:16.015;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:14:22.427;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:14:22.577;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:14:23.676;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:14:28.374;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:14:40.211;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:14:44.290;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:15:00.739;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:15:01.078;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:15:07.486;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:15:07.639;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:15:08.740;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:15:13.422;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:15:25.274;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:15:29.356;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:15:45.805;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:15:46.142;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:15:52.549;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:15:52.712;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:15:53.810;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:15:58.481;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:16:10.334;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:16:14.405;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:16:30.870;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:16:31.199;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:16:37.608;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:16:37.785;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:16:38.878;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:16:43.534;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:16:55.386;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:16:59.458;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:17:15.928;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:17:16.266;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:17:22.673;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:17:22.856;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:17:23.947;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:17:28.597;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:17:40.448;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:17:44.519;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:18:00.989;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:18:01.335;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:18:07.778;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:18:07.926;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:18:08.014;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:18:13.660;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:18:25.504;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:18:29.578;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:18:45.055;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:18:46.405;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:18:52.842;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:18:52.996;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:18:53.080;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:18:58.716;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:19:10.562;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:19:11.072;02;PBS_Server.6171;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:19:14.622;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:19:30.121;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:19:31.468;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:19:37.068;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:19:37.904;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:19:38.153;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:19:43.772;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:19:55.621;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:19:59.681;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:20:15.186;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:20:16.532;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:20:22.129;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:20:22.969;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:20:23.222;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:20:28.829;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:20:40.686;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:20:44.739;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:21:00.247;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:21:01.598;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:21:07.043;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:21:07.201;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:21:08.280;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:21:13.879;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:21:25.742;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:21:29.798;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:21:45.308;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:21:46.664;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:21:52.108;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:21:52.275;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:21:53.346;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:21:58.933;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:22:10.806;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:22:14.860;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:22:30.373;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:22:31.734;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:22:37.190;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:22:37.343;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:22:38.412;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:22:43.983;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:22:55.861;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:22:59.920;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:23:15.435;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:23:16.796;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:23:22.258;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:23:22.408;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:23:23.483;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:23:28.039;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:23:40.923;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:23:44.980;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:24:00.500;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:24:01.866;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:24:07.312;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:24:07.475;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:24:08.553;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:24:11.164;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:24:13.097;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:24:25.986;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:24:29.069;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:24:45.564;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:24:46.935;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:24:52.397;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:24:52.542;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:24:53.622;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:24:58.154;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:25:10.043;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:25:14.140;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:25:30.629;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:25:31.972;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:25:37.450;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:25:37.612;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:25:38.695;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:25:43.204;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:25:55.109;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:25:59.208;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:26:15.684;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:26:16.046;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:26:22.512;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:26:22.683;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:26:23.756;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:26:28.262;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:26:40.171;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:26:44.267;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:27:00.740;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:27:01.116;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:27:07.577;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:27:07.750;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:27:08.818;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:27:13.313;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:27:25.233;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:27:29.326;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:27:45.813;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:27:46.166;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:27:52.656;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:27:52.822;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:27:53.883;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:27:58.370;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:28:10.293;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:28:14.386;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:28:30.880;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:28:31.235;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:28:37.721;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:28:37.892;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:28:38.952;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:28:43.427;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:28:55.358;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:28:59.447;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:29:11.011;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:29:15.949;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:29:16.307;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:29:22.786;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:29:22.962;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:29:23.025;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:29:28.483;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:29:40.412;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:29:44.570;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:30:01.011;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:30:01.373;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:30:07.017;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:30:07.851;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:30:08.098;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:30:13.533;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:30:25.455;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:30:29.628;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:30:46.076;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:30:46.432;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:30:52.082;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:30:52.907;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:30:53.168;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:30:58.579;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:31:10.514;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:31:14.687;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:31:31.140;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:31:31.500;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:31:37.146;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:31:37.963;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:31:38.236;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:31:43.633;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:31:55.573;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:31:59.745;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:32:16.203;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:32:16.566;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:32:22.029;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:32:22.219;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:32:23.306;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:32:28.683;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:32:40.636;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:32:44.827;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:33:01.246;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:33:01.675;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:33:07.091;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:33:07.289;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:33:08.372;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:33:13.740;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:33:25.699;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:33:29.889;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:33:46.301;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:33:46.740;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:33:52.155;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:33:52.358;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:33:53.437;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:33:58.798;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:34:10.761;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:34:11.104;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:34:14.947;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:34:31.368;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:34:31.809;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:34:37.210;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:34:37.430;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:34:38.508;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:34:43.855;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:34:55.821;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:35:00.014;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:35:16.431;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:35:16.880;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:35:22.272;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:35:22.498;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:35:23.580;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:35:28.912;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:35:40.890;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:35:44.082;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:36:01.478;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:36:01.949;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:36:07.332;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:36:07.562;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:36:08.646;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:36:13.966;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:36:25.954;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:36:29.136;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:36:46.541;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:36:47.015;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:36:52.391;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:36:52.632;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:36:53.729;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:36:58.014;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:37:10.018;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:37:14.197;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:37:31.084;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:37:31.605;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:37:37.448;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:37:37.702;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:37:38.799;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:37:43.071;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:37:55.081;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:37:59.257;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:38:16.185;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:38:16.666;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:38:22.518;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:38:22.764;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:38:23.855;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:38:28.131;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:38:40.144;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:38:44.346;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:39:01.252;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:39:01.729;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:39:07.579;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:39:07.834;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:39:08.925;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:39:11.196;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:39:13.183;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:39:25.204;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:39:29.396;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:39:46.320;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:39:46.793;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:39:52.641;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:39:52.905;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:39:53.994;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:39:58.240;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:40:10.263;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:40:14.462;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:40:31.386;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:40:31.850;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:40:37.706;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:40:37.974;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:40:38.062;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:40:43.296;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:40:55.328;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:40:59.510;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:41:16.455;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:41:16.910;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:41:22.039;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:41:22.770;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:41:23.131;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:41:28.347;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:41:40.392;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:41:44.595;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:42:01.516;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:42:01.979;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:42:07.114;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:42:07.838;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:42:08.200;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:42:13.404;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:42:25.451;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:42:29.655;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:42:46.039;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:42:46.586;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:42:52.185;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:42:52.900;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:42:53.280;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:42:58.466;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:43:10.512;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:43:14.724;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:43:31.107;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:43:31.659;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:43:37.252;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:43:37.982;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:43:38.350;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:43:43.513;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:43:55.574;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:43:59.790;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:44:11.037;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:44:16.171;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:44:16.723;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:44:22.044;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:44:22.326;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:44:23.420;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:44:28.569;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:44:40.628;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:44:44.857;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:45:01.229;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:45:01.793;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:45:07.117;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:45:07.390;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:45:08.499;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:45:13.626;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:45:25.692;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:45:29.915;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:45:46.292;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:45:46.859;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:45:52.179;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:45:52.456;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:45:53.569;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:45:58.683;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:46:10.757;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:46:14.976;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:46:31.350;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:46:31.958;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:46:37.236;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:46:37.525;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:46:38.659;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:46:43.733;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:46:55.821;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:46:59.039;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:47:16.035;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:47:16.416;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:47:22.298;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:47:22.595;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:47:23.773;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:47:28.783;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:47:40.880;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:47:44.104;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:48:01.103;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:48:01.470;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:48:07.366;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:48:07.639;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:48:08.846;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:48:13.837;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:48:25.942;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:48:29.165;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:48:46.164;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:48:46.519;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:48:52.431;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:48:52.702;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:48:53.912;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:48:58.909;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:49:11.008;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:49:11.130;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:49:14.256;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:49:31.219;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:49:31.577;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:49:37.506;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:49:37.772;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:49:38.957;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:49:43.967;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:49:55.071;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:49:59.329;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:50:16.284;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:50:16.649;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:50:22.569;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:50:22.840;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:50:23.022;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:50:28.023;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:50:40.137;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:50:44.388;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:51:01.351;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:51:01.712;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:51:07.630;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:51:07.915;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:51:08.096;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:51:13.080;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:51:25.201;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:51:29.446;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:51:46.419;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:51:46.777;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:51:52.694;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:51:52.996;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:51:53.154;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:51:58.161;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:52:10.254;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:52:14.506;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:52:31.488;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:52:31.840;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:52:37.065;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:52:37.759;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:52:38.223;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:52:43.216;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:52:55.312;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:52:59.564;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:53:16.558;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:53:16.900;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:53:22.135;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:53:22.818;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:53:23.292;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:53:28.274;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:53:40.377;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:53:44.606;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:54:01.643;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:54:01.964;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:54:07.204;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:54:07.880;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:54:08.365;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:54:11.221;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:54:13.372;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:54:25.441;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:54:29.669;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:54:46.028;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:54:46.713;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:54:52.273;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:54:52.946;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:54:53.459;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:54:58.430;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:55:10.503;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:55:14.735;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:55:31.092;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:55:31.783;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:55:37.027;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:55:37.343;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:55:38.526;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:55:43.504;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:55:55.561;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:55:59.798;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:56:16.153;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:56:16.852;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:56:22.095;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:56:22.413;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:56:23.589;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:56:28.571;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:56:40.626;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:56:44.855;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:57:01.214;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:57:01.921;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:57:07.160;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:57:07.488;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:57:08.681;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:57:13.633;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:57:25.691;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:57:29.918;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:57:46.273;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:57:46.988;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:57:52.219;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:57:52.554;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:57:53.769;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:57:58.683;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:58:10.754;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:58:14.974;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:58:31.053;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:58:31.329;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:58:37.283;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:58:37.629;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:58:38.843;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:58:43.741;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:58:55.819;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:58:59.034;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 10:59:11.063;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 10:59:16.125;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 10:59:16.389;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 10:59:22.339;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 10:59:22.699;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 10:59:23.926;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 10:59:28.797;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 10:59:40.878;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 10:59:44.094;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:00:01.192;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:00:01.453;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:00:07.404;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:00:07.769;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:00:09.015;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:00:13.855;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:00:25.943;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:00:29.172;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:00:46.258;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:00:46.509;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:00:52.466;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:00:52.832;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:00:54.072;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:00:58.913;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:01:11.016;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:01:14.231;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:01:31.319;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:01:31.574;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:01:37.521;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:01:37.901;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:01:39.152;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:01:43.970;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:01:56.081;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:01:59.285;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:02:16.383;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:02:16.639;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:02:22.579;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:02:22.970;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:02:24.229;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:02:28.026;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:02:41.144;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:02:44.359;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:03:01.443;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:03:01.699;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:03:07.039;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:03:07.631;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:03:09.309;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:03:13.084;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:03:26.200;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:03:29.456;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:03:46.514;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:03:46.750;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:03:52.110;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:03:52.695;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:03:54.382;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:03:58.142;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:04:11.154;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:04:11.264;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:04:14.516;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:04:31.577;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:04:31.814;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:04:37.175;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:04:37.756;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:04:39.458;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:04:43.194;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:04:56.322;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:04:59.576;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:05:16.655;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:05:16.870;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:05:22.240;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:05:22.818;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:05:24.527;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:05:28.245;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:05:41.391;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:05:44.628;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:06:01.726;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:06:01.933;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:06:07.312;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:06:07.883;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:06:09.593;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:06:13.293;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:06:26.454;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:06:29.688;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:06:46.793;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:06:46.990;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:06:52.383;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:06:52.940;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:06:54.660;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:06:58.343;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:07:11.515;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:07:14.767;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:07:31.053;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:07:31.867;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:07:37.450;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:07:37.994;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:07:39.720;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:07:43.404;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:07:56.573;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:07:59.828;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:08:16.103;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:08:16.934;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:08:22.058;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:08:22.526;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:08:24.792;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:08:28.452;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:08:41.640;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:08:44.919;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:09:01.165;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:09:01.004;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:09:07.119;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:09:07.610;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:09:09.857;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:09:11.248;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:09:13.506;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:09:26.751;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:09:29.973;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:09:46.071;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:09:46.228;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:09:52.186;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:09:52.682;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:09:54.928;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:09:58.565;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:10:11.831;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:10:14.044;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:10:31.141;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:10:31.293;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:10:37.251;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:10:37.742;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:10:39.998;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:10:43.622;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:10:56.902;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:10:59.108;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:11:16.217;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:11:16.353;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:11:22.316;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:11:22.809;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:11:24.068;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:11:28.688;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:11:41.956;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:11:44.167;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:12:01.285;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:12:01.415;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:12:07.381;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:12:07.876;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:12:09.137;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:12:13.745;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:12:26.040;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:12:29.226;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:12:46.354;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:12:46.473;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:12:52.440;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:12:52.989;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:12:54.207;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:12:58.799;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:13:11.106;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:13:14.285;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:13:31.413;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:13:31.529;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:13:37.052;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:13:37.501;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:13:39.269;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:13:43.861;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:13:56.171;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:13:59.345;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:14:11.089;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:14:16.472;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:14:16.593;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:14:22.124;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:14:22.565;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:14:24.340;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:14:28.920;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:14:41.228;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:14:44.399;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:15:01.530;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:15:01.660;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:15:07.181;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:15:07.630;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:15:09.409;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:15:13.974;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:15:26.287;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:15:29.454;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:15:46.593;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:15:46.723;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:15:52.247;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:15:52.689;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:15:54.478;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:15:58.033;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:16:11.353;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:16:14.525;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:16:31.657;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:16:31.786;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:16:37.317;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:16:37.754;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:16:39.534;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:16:43.085;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:16:56.420;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:16:59.594;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:17:16.721;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:17:16.847;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:17:22.387;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:17:22.815;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:17:24.594;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:17:28.133;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:17:41.481;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:17:44.662;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:18:01.789;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:18:01.905;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:18:07.450;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:18:07.870;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:18:09.656;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:18:13.179;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:18:26.543;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:18:29.757;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:18:46.857;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:18:46.966;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:18:52.545;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:18:52.928;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:18:54.723;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:18:58.227;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:19:11.181;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:19:11.608;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:19:14.833;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:19:31.030;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:19:31.926;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:19:37.615;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:19:37.986;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:19:39.796;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:19:43.284;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:19:56.663;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:19:59.896;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:20:16.090;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:20:16.990;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:20:22.046;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:20:22.679;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:20:24.867;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:20:28.340;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:20:41.721;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:20:44.958;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:21:01.055;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:21:01.151;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:21:07.127;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:21:07.753;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:21:09.938;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:21:13.397;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:21:26.785;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:21:30.019;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:21:46.110;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:21:46.213;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:21:52.192;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:21:52.823;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:21:55.012;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:21:58.456;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:22:11.849;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:22:15.076;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:22:31.180;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:22:31.278;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:22:37.267;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:22:37.885;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:22:39.072;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:22:43.513;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:22:56.912;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:23:00.142;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:23:16.249;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:23:16.335;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:23:22.332;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:23:22.958;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:23:24.142;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:23:28.570;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:23:41.971;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:23:45.212;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:24:01.318;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:24:01.396;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:24:07.028;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:24:07.391;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:24:09.206;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:24:11.027;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:24:13.627;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:24:26.034;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:24:30.299;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:24:46.387;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:24:46.461;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:24:52.064;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:24:52.455;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:24:54.275;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:24:58.684;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:25:11.093;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:25:15.360;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:25:31.454;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:25:31.517;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:25:37.132;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:25:37.516;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:25:39.335;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:25:43.740;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:25:56.159;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:26:00.420;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:26:16.518;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:26:16.577;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:26:22.199;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:26:22.577;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:26:24.397;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:26:28.797;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:26:41.214;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:26:45.481;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:27:01.585;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:27:01.630;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:27:07.273;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:27:07.641;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:27:09.472;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:27:13.854;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:27:26.269;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:27:30.525;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:27:46.622;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:27:46.691;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:27:52.345;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:27:52.699;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:27:54.557;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:27:58.916;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:28:11.330;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:28:15.588;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:28:31.690;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:28:31.750;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:28:37.414;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:28:37.761;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:28:39.626;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:28:43.969;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:28:56.395;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:29:00.633;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:29:11.117;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:29:16.764;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:29:16.816;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:29:22.474;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:29:22.818;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:29:24.697;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:29:28.027;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:29:41.449;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:29:45.692;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:30:01.832;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:30:01.880;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:30:07.540;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:30:07.879;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:30:09.779;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:30:13.083;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:30:26.510;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:30:30.760;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:30:46.897;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:30:46.944;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:30:52.612;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:30:52.965;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:30:54.856;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:30:58.157;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:31:11.572;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:31:15.824;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:31:31.962;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:31:31.000;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:31:37.029;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:31:37.682;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:31:39.932;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:31:43.206;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:31:56.636;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:32:00.878;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:32:16.026;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:32:16.064;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:32:22.123;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:32:22.751;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:32:25.022;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:32:28.264;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:32:41.700;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:32:45.953;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:33:01.086;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:33:01.131;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:33:07.205;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:33:07.825;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:33:09.103;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:33:13.324;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:33:26.765;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:33:31.012;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:33:46.155;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:33:46.183;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:33:52.272;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:33:52.894;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:33:54.168;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:33:58.373;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:34:11.213;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:34:11.828;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:34:15.078;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:34:31.226;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:34:31.239;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:34:37.333;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:34:37.003;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:34:39.237;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:34:43.429;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:34:56.881;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:35:00.157;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:35:16.295;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:35:16.303;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:35:22.070;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:35:22.390;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:35:24.307;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:35:28.492;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:35:41.952;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:35:45.217;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:36:01.356;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:36:01.362;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:36:07.143;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:36:07.449;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:36:09.372;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:36:13.546;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:36:26.020;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:36:30.275;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:36:46.419;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:36:46.430;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:36:52.215;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:36:52.511;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:36:54.442;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:36:58.603;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:37:11.080;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:37:15.335;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:37:31.479;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:37:31.500;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:37:37.282;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:37:37.590;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:37:39.511;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:37:43.662;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:37:56.141;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:38:00.385;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:38:16.554;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:38:16.572;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:38:22.347;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:38:22.630;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:38:24.585;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:38:28.716;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:38:41.199;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:38:45.447;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:39:01.617;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:39:01.646;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:39:07.452;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:39:07.706;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:39:09.660;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:39:11.058;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:39:13.775;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:39:26.265;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:39:30.522;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:39:46.688;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:39:46.717;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:39:52.523;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:39:52.779;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:39:54.741;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:39:58.834;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:40:11.324;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:40:15.589;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:40:31.751;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:40:31.759;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:40:37.591;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:40:37.858;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:40:39.811;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:40:43.894;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:40:56.388;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:41:00.654;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:41:16.811;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:41:16.852;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:41:22.657;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:41:22.920;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:41:24.880;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:41:28.947;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:41:41.453;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:41:45.718;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:42:01.876;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:42:01.917;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:42:07.724;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:42:07.986;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:42:09.979;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:42:13.996;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:42:26.507;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:42:30.778;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:42:46.945;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:42:46.987;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:42:52.047;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:42:52.794;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:42:54.055;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:42:58.055;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:43:11.560;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:43:15.837;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:43:31.057;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:43:31.002;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:43:37.119;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:43:37.864;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:43:39.127;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:43:43.107;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:43:56.623;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:44:00.891;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:44:11.156;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:44:16.066;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:44:16.162;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:44:22.187;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:44:22.932;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:44:24.196;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:44:28.163;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:44:41.678;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:44:45.946;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:45:01.133;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:45:01.235;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:45:07.252;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:45:07.998;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:45:09.266;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:45:13.217;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:45:26.744;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:45:31.018;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:45:46.189;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:45:46.305;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:45:52.063;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:45:52.309;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:45:54.337;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:45:58.278;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:46:11.808;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:46:16.079;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:46:31.250;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:46:31.372;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:46:37.128;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:46:37.368;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:46:39.403;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:46:43.333;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:46:56.880;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:47:01.138;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:47:16.313;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:47:16.432;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:47:22.191;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:47:22.441;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:47:24.483;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:47:28.385;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:47:41.946;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:47:46.200;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:48:01.375;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:48:01.504;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:48:07.251;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:48:07.514;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:48:09.556;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:48:13.444;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:48:26.010;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:48:31.268;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:48:46.434;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:48:46.577;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:48:52.319;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:48:52.580;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:48:54.625;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:48:58.495;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:49:11.069;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:49:11.247;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:49:16.328;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:49:31.489;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:49:31.646;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:49:37.382;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:49:37.638;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:49:39.695;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:49:43.546;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:49:56.131;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:50:01.387;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:50:16.552;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:50:16.716;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:50:22.444;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:50:22.701;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:50:24.765;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:50:28.603;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:50:41.200;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:50:46.445;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:51:01.618;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:51:01.795;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:51:07.501;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:51:07.766;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:51:09.839;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:51:13.661;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:51:26.264;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:51:31.508;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:51:46.678;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:51:46.855;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:51:52.580;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:51:52.831;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:51:54.937;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:51:58.731;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:52:11.330;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:52:16.569;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:52:31.737;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:52:31.927;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:52:37.643;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:52:37.885;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:52:39.006;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:52:43.788;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:52:56.392;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:53:01.628;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:53:16.794;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:53:16.000;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:53:22.713;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:53:22.946;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:53:24.074;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:53:28.846;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:53:41.475;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:53:46.693;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:54:01.067;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:54:01.856;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:54:07.782;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:54:08.008;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:54:09.147;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:54:11.087;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:54:13.924;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:54:26.532;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:54:31.755;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:54:46.136;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:54:46.919;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:54:52.071;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:54:52.862;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:54:54.220;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:54:58.974;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:55:11.598;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:55:16.814;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:55:31.197;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:55:31.978;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:55:37.129;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:55:37.956;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:55:39.313;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:55:43.029;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:55:56.662;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:56:01.873;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:56:16.040;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:56:16.268;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:56:22.193;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:56:23.020;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:56:24.383;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:56:28.102;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:56:41.739;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:56:46.935;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:57:01.103;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:57:01.337;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:57:07.257;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:57:08.089;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:57:09.455;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:57:13.154;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:57:26.800;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:57:31.996;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:57:46.168;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:57:46.407;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:57:52.320;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:57:53.153;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:57:54.526;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:57:58.204;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:58:11.852;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:58:16.016;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:58:31.228;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:58:31.469;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:58:37.385;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:58:38.224;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:58:39.598;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:58:43.261;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:58:56.920;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:59:01.076;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 11:59:11.181;02;PBS_Server.6181;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 11:59:16.290;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 11:59:16.539;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 11:59:22.449;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 11:59:23.306;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 11:59:24.658;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 11:59:28.324;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 11:59:41.977;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 11:59:46.137;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:00:01.349;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:00:01.605;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:00:07.510;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:00:08.369;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:00:09.732;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:00:13.385;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:00:26.037;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:00:31.188;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:00:46.415;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:00:46.671;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:00:52.568;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:00:53.436;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:00:54.802;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:00:58.433;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:01:11.102;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:01:16.248;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:01:31.477;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:01:31.729;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:01:37.620;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:01:38.501;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:01:39.858;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:01:43.497;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:01:56.167;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:02:01.299;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:02:16.541;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:02:16.797;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:02:22.679;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:02:23.573;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:02:24.934;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:02:28.547;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:02:41.231;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:02:46.396;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:03:01.599;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:03:01.866;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:03:07.734;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:03:08.634;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:03:10.010;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:03:13.604;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:03:26.296;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:03:31.456;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:03:46.654;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:03:46.936;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:03:52.799;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:03:53.731;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:03:54.076;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:03:58.655;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:04:11.023;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:04:11.361;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:04:16.507;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:04:31.719;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:04:31.007;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:04:37.857;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:04:38.800;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:04:39.154;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:04:43.707;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:04:56.420;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:05:01.567;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:05:16.060;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:05:16.785;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:05:22.927;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:05:23.863;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:05:24.224;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:05:28.764;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:05:41.481;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:05:46.630;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:06:01.126;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:06:01.845;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:06:07.992;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:06:08.976;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:06:09.289;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:06:13.821;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:06:26.541;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:06:31.692;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:06:46.186;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:06:46.901;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:06:52.056;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:06:53.045;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:06:54.359;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:06:58.873;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:07:11.604;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:07:16.755;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:07:31.256;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:07:31.966;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:07:37.108;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:07:38.114;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:07:39.431;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:07:43.926;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:07:56.670;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:08:01.815;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:08:16.022;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:08:16.327;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:08:22.172;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:08:23.203;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:08:24.504;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:08:28.981;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:08:41.732;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:08:46.895;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:09:01.085;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:09:01.395;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:09:07.219;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:09:08.285;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:09:09.565;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:09:11.113;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:09:13.036;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:09:26.798;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:09:31.964;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:09:46.149;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:09:46.465;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:09:52.283;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:09:53.360;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:09:54.648;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:09:58.095;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:10:11.862;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:10:16.040;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:10:31.217;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:10:31.536;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:10:37.347;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:10:38.431;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:10:39.725;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:10:43.152;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:10:56.926;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:11:01.105;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:11:16.278;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:11:16.603;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:11:22.411;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:11:23.497;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:11:24.787;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:11:28.205;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:11:41.981;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:11:46.142;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:12:01.340;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:12:01.672;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:12:07.475;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:12:08.588;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:12:09.854;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:12:13.258;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:12:26.044;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:12:31.216;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:12:46.400;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:12:46.744;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:12:52.529;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:12:53.664;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:12:54.931;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:12:58.316;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:13:11.100;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:13:16.275;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:13:31.460;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:13:31.814;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:13:37.596;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:13:38.749;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:13:39.000;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:13:43.366;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:13:56.159;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:14:01.335;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:14:11.206;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:14:16.517;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:14:16.884;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:14:22.649;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:14:23.821;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:14:24.079;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:14:28.424;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:14:41.218;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:14:46.398;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:15:01.584;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:15:01.951;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:15:07.714;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:15:08.888;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:15:09.149;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:15:13.482;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:15:26.282;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:15:31.461;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:15:46.640;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:15:47.015;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:15:52.797;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:15:53.968;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:15:54.218;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:15:58.536;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:16:11.346;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:16:16.518;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:16:31.077;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:16:31.699;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:16:37.859;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:16:38.037;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:16:39.279;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:16:43.597;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:16:56.410;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:17:01.579;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:17:16.156;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:17:16.764;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:17:22.918;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:17:23.108;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:17:24.353;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:17:28.655;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:17:41.470;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:17:46.639;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:18:01.227;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:18:01.827;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:18:07.975;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:18:08.175;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:18:09.422;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:18:13.706;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:18:26.532;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:18:31.742;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:18:46.296;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:18:46.884;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:18:52.039;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:18:53.241;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:18:54.489;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:18:58.763;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:19:11.055;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:19:11.597;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:19:16.802;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:19:31.367;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:19:31.950;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:19:37.110;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:19:38.316;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:19:39.553;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:19:43.821;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:19:56.650;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:20:01.862;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:20:16.427;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:20:17.011;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:20:22.166;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:20:23.380;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:20:24.623;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:20:28.892;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:20:41.710;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:20:46.923;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:21:01.502;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:21:02.071;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:21:07.240;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:21:08.450;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:21:09.681;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:21:13.946;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:21:26.771;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:21:31.016;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:21:46.567;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:21:47.129;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:21:52.305;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:21:53.514;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:21:54.750;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:21:58.989;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:22:11.836;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:22:16.078;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:22:31.664;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:22:32.188;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:22:37.380;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:22:38.584;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:22:39.818;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:22:43.035;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:22:56.900;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:23:01.139;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:23:16.745;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:23:17.245;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:23:22.443;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:23:23.654;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:23:24.889;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:23:28.092;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:23:41.961;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:23:46.194;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:24:01.817;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:24:02.306;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:24:07.505;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:24:08.762;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:24:09.959;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:24:11.148;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:24:13.151;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:24:26.024;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:24:31.264;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:24:46.916;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:24:47.359;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:24:52.566;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:24:53.830;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:24:54.023;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:24:58.200;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:25:11.082;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:25:16.324;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:25:31.985;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:25:32.424;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:25:37.629;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:25:38.889;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:25:39.090;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:25:43.256;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:25:56.148;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:26:01.389;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:26:16.057;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:26:17.478;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:26:22.689;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:26:23.955;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:26:24.157;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:26:28.333;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:26:41.205;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:26:46.450;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:27:01.128;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:27:02.544;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:27:07.749;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:27:09.023;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:27:09.227;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:27:13.427;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:27:26.257;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:27:31.507;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:27:46.202;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:27:47.608;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:27:52.809;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:27:54.085;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:27:54.295;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:27:58.502;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:28:11.319;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:28:16.569;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:28:31.250;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:28:32.669;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:28:37.866;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:28:39.191;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:28:39.363;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:28:43.559;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:28:56.383;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:29:01.629;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:29:11.245;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:29:16.327;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:29:17.728;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:29:22.920;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:29:24.261;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:29:24.434;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:29:28.619;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:29:41.438;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:29:46.689;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:30:01.398;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:30:02.792;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:30:07.985;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:30:09.331;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:30:09.504;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:30:13.684;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:30:26.497;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:30:31.747;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:30:46.447;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:30:47.849;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:30:52.047;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:30:54.397;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:30:54.573;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:30:58.744;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:31:11.562;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:31:16.813;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:31:31.511;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:31:32.903;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:31:37.108;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:31:39.464;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:31:39.642;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:31:43.802;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:31:56.626;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:32:01.883;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:32:16.580;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:32:17.968;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:32:22.174;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:32:24.536;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:32:24.713;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:32:28.858;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:32:41.694;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:32:46.968;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:33:01.648;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:33:02.028;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:33:07.239;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:33:09.610;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:33:09.779;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:33:13.926;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:33:26.753;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:33:31.033;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:33:46.715;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:33:47.088;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:33:52.304;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:33:54.685;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:33:54.847;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:33:58.989;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:34:11.089;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:34:11.818;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:34:16.096;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:34:31.793;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:34:32.152;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:34:37.355;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:34:39.756;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:34:39.916;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:34:43.044;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:34:56.892;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:35:01.156;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:35:16.875;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:35:17.212;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:35:22.419;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:35:24.828;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:35:24.987;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:35:28.101;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:35:41.955;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:35:46.216;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:36:01.948;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:36:02.273;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:36:07.483;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:36:09.040;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:36:09.897;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:36:13.159;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:36:26.020;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:36:31.271;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:36:47.012;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:36:47.337;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:36:52.554;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:36:54.109;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:36:54.968;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:36:58.242;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:37:11.086;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:37:16.315;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:37:32.084;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:37:32.402;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:37:37.626;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:37:39.037;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:37:39.181;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:37:43.301;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:37:56.149;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:38:01.379;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:38:17.155;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:38:17.454;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:38:22.715;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:38:24.109;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:38:24.254;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:38:28.358;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:38:41.210;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:38:46.457;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:39:02.223;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:39:02.512;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:39:07.779;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:39:09.180;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:39:09.323;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:39:11.181;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:39:13.415;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:39:26.270;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:39:31.517;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:39:47.294;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:39:47.569;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:39:52.843;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:39:54.250;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:39:54.393;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:39:58.476;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:40:11.335;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:40:16.578;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:40:32.365;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:40:32.627;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:40:37.898;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:40:39.313;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:40:39.459;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:40:43.543;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:40:56.388;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:41:01.639;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:41:17.431;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:41:17.689;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:41:22.962;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:41:24.379;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:41:24.528;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:41:28.613;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:41:41.455;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:41:46.699;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:42:02.510;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:42:02.751;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:42:07.040;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:42:09.445;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:42:09.598;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:42:13.674;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:42:26.507;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:42:31.758;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:42:47.580;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:42:47.816;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:42:52.099;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:42:54.514;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:42:54.667;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:42:58.735;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:43:11.562;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:43:16.815;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:43:32.649;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:43:32.879;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:43:37.165;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:43:39.586;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:43:39.736;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:43:43.794;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:43:56.626;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:44:01.879;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:44:11.023;02;PBS_Server.6167;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:44:17.725;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:44:17.945;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:44:22.232;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:44:24.653;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:44:24.806;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:44:28.861;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:44:41.687;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:44:46.941;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:45:02.792;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:45:03.010;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:45:07.306;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:45:09.717;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:45:09.876;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:45:13.907;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:45:26.750;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:45:31.003;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:45:47.858;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:45:48.076;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:45:52.369;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:45:54.782;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:45:54.946;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:45:58.956;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:46:11.816;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:46:16.062;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:46:32.922;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:46:33.141;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:46:37.431;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:46:39.851;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:46:40.015;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:46:43.024;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:46:56.876;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:47:01.115;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:47:17.990;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:47:18.206;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:47:22.490;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:47:24.921;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:47:25.084;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:47:28.083;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:47:41.942;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:47:46.189;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:48:02.054;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:48:03.260;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:48:07.539;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:48:09.974;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:48:10.149;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:48:13.139;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:48:26.007;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:48:31.249;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:48:47.123;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:48:48.313;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:48:52.630;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:48:54.044;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:48:55.208;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:48:58.194;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:49:11.073;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:49:11.113;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:49:16.317;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:49:32.186;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:49:33.368;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:49:37.692;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:49:39.111;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:49:40.277;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:49:43.254;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:49:56.135;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:50:01.372;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:50:17.250;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:50:18.430;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:50:22.755;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:50:24.183;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:50:25.347;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:50:28.311;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:50:41.200;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:50:46.428;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:51:02.315;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:51:03.475;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:51:07.831;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:51:09.292;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:51:10.417;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:51:13.369;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:51:26.266;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:51:31.489;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:51:47.385;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:51:48.529;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:51:52.891;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:51:54.362;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:51:55.486;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:51:58.425;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:52:11.331;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:52:16.548;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:52:32.483;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:52:33.594;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:52:37.944;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:52:39.432;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:52:40.574;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:52:43.482;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:52:56.395;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:53:01.602;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:53:17.554;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:53:18.646;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:53:22.998;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:53:24.502;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:53:25.644;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:53:28.534;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:53:41.458;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:53:46.662;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:54:02.624;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:54:03.710;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:54:07.059;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:54:09.577;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:54:10.714;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:54:11.206;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:54:13.582;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:54:26.520;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:54:31.723;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:54:47.692;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:54:48.774;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:54:52.120;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:54:54.647;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:54:55.786;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:54:58.641;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:55:11.581;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:55:16.783;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:55:32.780;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:55:33.837;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:55:37.185;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:55:39.711;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:55:40.855;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:55:43.693;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:55:56.648;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:56:01.842;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:56:17.849;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:56:18.905;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:56:22.257;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:56:24.777;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:56:25.917;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:56:28.747;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:56:41.710;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:56:46.898;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:57:02.921;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:57:03.958;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:57:07.325;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:57:09.891;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:57:10.989;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:57:13.804;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:57:26.771;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:57:31.955;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:57:47.990;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:57:48.022;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:57:52.382;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:57:54.961;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:57:55.066;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:57:58.858;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:58:11.832;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:58:16.013;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:58:32.052;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:58:33.079;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:58:37.449;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:58:39.029;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:58:40.135;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:58:43.916;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:58:56.896;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:59:01.082;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 12:59:11.046;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 12:59:17.124;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 12:59:18.140;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 12:59:22.512;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 12:59:24.097;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 12:59:25.203;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 12:59:28.966;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 12:59:41.950;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 12:59:46.142;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:00:02.188;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:00:03.202;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:00:07.572;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:00:09.159;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:00:10.271;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:00:13.013;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:00:27.011;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:00:31.202;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:00:47.254;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:00:48.260;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:00:52.619;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:00:54.258;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:00:55.341;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:00:58.064;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:01:12.075;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:01:16.262;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:01:32.318;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:01:33.320;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:01:37.684;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:01:39.328;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:01:40.409;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:01:43.128;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:01:57.138;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:02:01.322;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:02:17.386;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:02:18.385;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:02:22.757;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:02:24.398;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:02:25.481;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:02:28.183;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:02:42.195;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:02:46.382;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:03:02.452;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:03:03.448;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:03:07.824;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:03:09.460;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:03:10.550;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:03:13.243;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:03:27.259;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:03:31.434;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:03:47.513;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:03:48.511;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:03:52.880;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:03:54.530;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:03:55.616;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:03:58.302;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:04:11.138;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 13:04:12.324;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:04:16.477;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:04:32.583;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:04:33.577;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:04:37.973;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:04:39.600;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:04:40.688;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:04:43.359;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:04:57.380;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:05:01.534;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:05:17.643;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:05:18.642;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:05:22.031;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:05:24.659;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:05:25.757;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:05:28.416;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:05:42.445;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:05:46.594;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:06:02.710;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:06:03.704;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:06:07.094;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:06:09.724;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:06:10.822;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:06:13.475;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:06:27.502;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:06:31.648;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:06:47.774;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:06:48.760;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:06:52.158;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:06:54.798;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:06:55.888;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:06:58.524;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:07:12.570;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:07:16.705;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:07:32.848;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:07:33.824;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:07:37.215;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:07:39.901;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:07:40.950;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:07:43.579;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:07:57.634;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:08:01.765;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:08:17.919;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:08:18.880;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:08:22.277;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:08:24.974;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:08:26.019;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:08:28.635;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:08:42.701;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:08:46.824;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:09:02.985;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:09:03.938;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:09:07.341;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:09:09.037;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:09:10.089;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:09:11.235;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 13:09:13.691;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:09:27.763;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:09:31.886;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:09:47.049;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:09:48.998;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:09:52.404;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:09:54.107;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:09:55.159;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:09:58.750;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:10:12.822;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:10:16.948;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:10:32.119;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:10:33.056;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:10:37.465;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:10:39.171;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:10:40.230;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:10:43.813;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:10:57.886;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:11:02.009;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:11:17.190;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:11:18.119;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:11:22.518;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:11:24.236;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:11:25.304;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:11:28.864;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:11:42.953;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:11:46.068;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:12:02.257;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:12:03.183;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:12:07.582;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:12:09.284;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:12:10.362;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:12:13.921;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:12:27.017;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:12:31.129;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:12:47.331;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:12:48.263;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:12:52.640;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:12:54.355;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:12:55.432;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:12:58.973;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:13:12.082;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:13:16.194;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:13:32.403;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:13:33.330;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:13:37.704;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:13:39.429;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:13:40.502;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:13:43.029;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:13:57.140;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:14:01.255;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:14:11.084;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 13:14:17.466;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:14:18.393;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:14:22.769;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:14:24.500;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:14:25.573;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:14:28.086;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:14:42.198;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:14:46.315;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:15:02.533;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:15:03.458;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:15:07.825;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:15:09.567;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:15:10.638;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:15:13.142;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:15:27.261;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:15:31.375;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:15:47.598;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:15:48.518;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:15:52.891;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:15:54.633;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:15:55.707;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:15:58.193;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:16:12.326;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:16:16.435;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:16:32.669;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:16:33.582;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:16:37.957;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:16:39.704;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:16:40.770;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:16:43.279;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:16:57.390;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:17:01.493;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:17:17.741;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:17:18.648;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:17:22.020;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:17:24.774;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:17:25.839;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:17:28.337;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:17:42.454;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:17:46.537;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:18:02.804;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:18:03.712;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:18:07.074;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:18:09.847;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:18:10.905;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:18:13.395;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:18:27.510;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:18:31.600;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:18:47.873;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:18:48.772;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:18:52.130;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:18:54.912;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:18:55.971;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:18:58.444;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:19:11.177;02;PBS_Server.6174;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 13:19:12.573;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:19:16.663;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:19:32.935;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:19:33.829;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:19:37.191;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:19:39.975;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:19:40.039;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:19:43.494;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:19:57.635;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:20:01.723;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:20:17.998;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:20:18.890;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:20:22.254;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:20:24.039;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:20:25.110;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:20:28.554;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:20:42.689;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:20:46.774;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:21:02.065;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:21:03.951;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:21:07.324;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:21:09.110;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:21:10.175;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:21:13.627;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:21:27.751;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:21:31.834;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:21:47.125;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:21:49.010;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:21:52.379;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:21:54.186;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:21:55.242;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:21:58.683;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:22:12.817;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:22:16.894;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:22:32.195;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:22:34.075;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:22:37.453;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:22:39.257;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:22:40.307;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:22:43.742;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:22:57.881;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:23:01.953;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:23:17.263;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:23:19.130;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:23:22.512;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:23:24.327;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:23:25.377;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:23:28.799;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:23:42.950;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:23:47.012;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:24:02.343;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:24:04.194;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:24:07.570;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:24:09.398;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:24:10.448;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:24:11.025;02;PBS_Server.6172;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 13:24:13.854;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:24:28.010;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:24:31.071;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:24:47.419;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:24:49.251;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:24:52.630;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:24:54.469;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:24:55.492;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:24:58.911;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:25:13.064;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:25:16.132;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:25:32.488;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:25:34.310;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:25:37.694;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:25:39.530;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:25:40.545;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:25:43.966;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:25:58.128;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:26:01.197;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:26:17.557;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:26:19.370;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:26:22.767;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:26:24.596;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:26:25.597;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:26:28.023;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:26:43.184;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:26:46.257;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:27:02.617;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:27:04.435;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:27:07.832;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:27:09.665;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:27:10.665;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:27:13.075;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:27:28.234;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:27:31.320;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:27:47.689;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:27:49.490;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:27:52.893;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:27:54.728;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:27:55.736;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:27:58.142;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:28:13.292;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:28:16.380;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:28:32.760;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:28:34.551;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:28:37.958;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:28:39.796;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:28:40.805;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:28:43.197;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:28:58.356;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:29:01.437;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:29:11.123;02;PBS_Server.6168;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 13:29:17.825;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:29:19.616;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:29:22.022;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:29:24.865;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:29:25.868;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:29:28.254;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:29:43.408;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:29:46.498;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:30:02.879;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:30:04.679;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:30:07.085;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:30:09.930;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:30:10.939;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:30:13.311;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:30:28.461;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:30:31.568;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:30:47.945;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:30:49.740;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:30:52.139;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:30:54.996;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:30:55.008;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:30:58.367;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:31:13.521;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:31:16.630;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:31:32.013;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:31:34.804;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:31:37.204;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:31:39.065;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:31:40.074;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:31:43.429;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:31:58.585;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:32:01.683;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:32:17.079;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:32:19.859;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:32:22.264;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:32:24.131;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:32:25.145;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:32:28.486;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:32:43.649;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:32:46.744;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:33:02.145;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:33:04.919;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:33:07.325;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:33:09.201;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:33:10.212;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:33:13.534;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:33:28.711;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:33:31.805;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:33:47.212;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:33:49.979;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:33:52.380;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:33:54.274;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:33:55.279;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:33:58.598;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:34:11.221;02;PBS_Server.6170;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 13:34:13.771;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:34:16.865;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:34:32.279;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:34:34.038;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:34:37.440;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:34:39.345;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:34:40.349;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:34:43.655;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:34:58.830;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:35:01.925;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:35:17.350;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:35:19.102;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:35:22.502;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:35:24.415;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:35:25.419;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:35:28.703;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:35:43.895;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:35:46.995;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:36:02.420;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:36:04.162;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:36:07.567;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:36:09.486;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:36:10.485;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:36:13.769;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:36:28.954;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:36:31.054;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:36:47.484;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:36:49.223;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:36:52.629;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:36:54.553;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:36:55.553;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:36:58.845;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:37:13.017;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:37:16.114;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:37:32.586;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:37:34.279;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:37:37.684;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:37:39.623;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:37:40.614;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:37:43.903;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:37:58.082;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:38:01.175;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:38:17.654;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:38:19.352;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:38:22.746;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:38:24.697;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:38:25.681;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:38:28.954;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:38:43.147;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:38:46.236;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:39:02.723;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:39:04.413;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:39:07.806;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:39:09.767;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:39:10.780;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:39:11.063;02;PBS_Server.6173;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 13:39:13.005;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:39:28.210;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:39:31.298;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:39:47.792;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:39:49.467;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:39:52.869;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:39:54.839;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:39:55.850;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:39:58.053;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:40:13.271;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:40:16.361;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:40:32.879;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:40:34.554;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:40:37.930;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:40:39.903;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:40:40.922;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:40:43.106;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:40:58.335;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:41:01.420;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:41:17.966;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:41:19.619;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:41:22.994;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:41:24.968;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:41:25.990;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:41:28.165;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:41:43.390;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:41:46.480;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:42:02.028;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:42:04.683;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:42:07.058;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:42:09.034;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:42:10.082;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:42:13.219;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:42:28.450;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:42:31.565;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:42:47.132;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:42:49.740;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:42:52.119;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:42:54.105;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:42:55.157;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:42:58.281;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:43:13.510;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:43:16.623;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:43:32.201;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:43:34.821;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:43:35.194;08;PBS_Server.6728;Job;svr_setjobstate;svr_setjobstate: setting job 625.master23.banette.gent.vsc state from TRANSIT-TRANSICM to QUEUED-QUEUED (1-10) +10/03/2016 13:43:35.194;256;PBS_Server.6728;Job;625.master23.banette.gent.vsc;enqueuing into default, state 1 hop 1 +10/03/2016 13:43:35.194;02;PBS_Server.6728;Svr;Act;Account file /var/spool/pbs/server_priv/accounting/20161003 opened +10/03/2016 13:43:35.195;256;PBS_Server.6728;Job;625.master23.banette.gent.vsc;dequeuing from default, state QUEUED +10/03/2016 13:43:35.195;256;PBS_Server.6728;Job;625.master23.banette.gent.vsc;enqueuing into long, state 1 hop 1 +10/03/2016 13:43:35.195;08;PBS_Server.6728;Job;req_commit;job_id: 625.master23.banette.gent.vsc +10/03/2016 13:43:35.196;02;PBS_Server.6728;node;close_conn;Closing connection 10 and calling its accompanying function on close +10/03/2016 13:43:37.183;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:43:39.178;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:43:40.228;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:43:43.338;04;PBS_Server.6210;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:43:45.334;08;PBS_Server.6183;Job;625.master23.banette.gent.vsc;Job Run at request of root@master23.banette.gent.vsc +10/03/2016 13:43:45.334;08;PBS_Server.6183;Job;svr_setjobstate;svr_setjobstate: setting job 625.master23.banette.gent.vsc state from QUEUED-QUEUED to RUNNING-PRERUN (4-40) +10/03/2016 13:43:50.862;02;PBS_Server.6183;Job;625.master23.banette.gent.vsc;child reported success for job after 5 seconds (dest=???), rc=0 +10/03/2016 13:43:50.862;08;PBS_Server.6183;Job;svr_setjobstate;svr_setjobstate: setting job 625.master23.banette.gent.vsc state from RUNNING-TRNOUTCM to RUNNING-RUNNING (4-42) +10/03/2016 13:43:50.863;08;PBS_Server.6183;Job;625.master23.banette.gent.vsc;Not sending email: job requested no e-mail +10/03/2016 13:43:58.572;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:44:01.684;04;PBS_Server.28039;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc +10/03/2016 13:44:11.153;02;PBS_Server.6169;Svr;PBS_Server;Torque Server Version = 6.0, loglevel = 2 +10/03/2016 13:44:17.268;04;PBS_Server.20703;Svr;svr_is_request;IS_STATUS received from node2802.banette.gent.vsc +10/03/2016 13:44:19.879;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2805.banette.gent.vsc +10/03/2016 13:44:22.248;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2807.banette.gent.vsc +10/03/2016 13:44:24.938;04;PBS_Server.6728;Svr;svr_is_request;IS_STATUS received from node2801.banette.gent.vsc +10/03/2016 13:44:25.299;04;PBS_Server.6182;Svr;svr_is_request;IS_STATUS received from node2804.banette.gent.vsc +10/03/2016 13:44:28.395;04;PBS_Server.20704;Svr;svr_is_request;IS_STATUS received from node2806.banette.gent.vsc +10/03/2016 13:44:33.218;09;PBS_Server.6210;Job;625.master23.banette.gent.vsc;obit received - updating final job usage info +10/03/2016 13:44:33.219;09;PBS_Server.6210;Job;625.master23.banette.gent.vsc;job exit status 143 handled +10/03/2016 13:44:33.219;08;PBS_Server.6210;Job;svr_setjobstate;svr_setjobstate: setting job 625.master23.banette.gent.vsc state from RUNNING-RUNNING to EXITING-EXITING (5-50) +10/03/2016 13:44:33.219;08;PBS_Server.6210;Job;625.master23.banette.gent.vsc;Not sending email: job requested no e-mail +10/03/2016 13:44:33.219;08;PBS_Server.6172;Job;625.master23.banette.gent.vsc;on_job_exit valid pjob: 625.master23.banette.gent.vsc (substate=50) +10/03/2016 13:44:33.219;08;PBS_Server.6172;Job;handle_exiting_or_abort_substate;625.master23.banette.gent.vsc; JOB_SUBSTATE_EXITING +10/03/2016 13:44:33.219;08;PBS_Server.6172;Job;svr_setjobstate;svr_setjobstate: setting job 625.master23.banette.gent.vsc state from EXITING-EXITING to EXITING-RETURNSTD (5-70) +10/03/2016 13:44:33.220;08;PBS_Server.6172;Job;svr_setjobstate;svr_setjobstate: setting job 625.master23.banette.gent.vsc state from EXITING-RETURNSTD to EXITING-STAGEOUT (5-52) +10/03/2016 13:44:35.160;08;PBS_Server.6172;Job;svr_setjobstate;svr_setjobstate: setting job 625.master23.banette.gent.vsc state from EXITING-STAGEOUT to EXITING-STAGEDEL (5-53) +10/03/2016 13:44:35.160;08;PBS_Server.6172;Job;svr_setjobstate;svr_setjobstate: setting job 625.master23.banette.gent.vsc state from EXITING-STAGEDEL to EXITING-EXITED (5-54) +10/03/2016 13:44:35.173;08;PBS_Server.6172;Job;svr_setjobstate;svr_setjobstate: setting job 625.master23.banette.gent.vsc state from EXITING-EXITED to COMPLETE-COMPLETE (6-59) +10/03/2016 13:44:35.174;16;PBS_Server.6172;Job;625.master23.banette.gent.vsc;Exit_status=143 resources_used.cput=1 resources_used.energy_used=0 resources_used.mem=51996kb resources_used.vmem=38060kb resources_used.walltime=00:00:42 +10/03/2016 13:44:35.174;256;PBS_Server.6172;Job;625.master23.banette.gent.vsc;dequeuing from long, state COMPLETE +10/03/2016 13:44:43.632;04;PBS_Server.28038;Svr;svr_is_request;IS_STATUS received from node2808.banette.gent.vsc +10/03/2016 13:44:46.743;04;PBS_Server.7306;Svr;svr_is_request;IS_STATUS received from node2803.banette.gent.vsc diff --git a/src/tools/test/tracejob/test_tracejob.c b/src/tools/test/tracejob/test_tracejob.c index 2e73feb30f..dd8c17b838 100644 --- a/src/tools/test/tracejob/test_tracejob.c +++ b/src/tools/test/tracejob/test_tracejob.c @@ -3,30 +3,130 @@ #include "test_tracejob.h" #include #include +#include +#include #include "pbs_error.h" -START_TEST(test_one) + + +// test with uncompressed log +START_TEST(test_20161003) { + struct tm *tm_ptr; + time_t t; + int file_count; + char *filenames[MAX_LOG_FILES_PER_DAY]; + const char *prefix_path = "."; + const char *jobid = "625"; + gzFile fp; + struct log_array log = {}; + unsigned int i; + + // create tm struct + t = time(NULL); + tm_ptr = localtime(&t); + ck_assert(tm_ptr != NULL); + + // set to 2016-10-03 + tm_ptr->tm_year = 2016 - 1900; + tm_ptr->tm_mon = 10 - 1; + tm_ptr->tm_mday = 3; + + file_count = log_path((char *)prefix_path, 1, tm_ptr, filenames); + + ck_assert_int_eq(file_count, 1); + ck_assert_str_eq(filenames[0], "./server_logs/20161003"); + + fp = gzopen(filenames[0], "r"); + ck_assert_msg(fp != NULL, "Failed to open ./server_logs/20161003 with gzopen"); + + ck_assert_int_eq(parse_log(&fp, (char *)jobid, 1, &log), 0); + + gzclose(fp); + free(filenames[0]); + // check first and one to last line + ck_assert_str_eq(log.log_lines[0].msg, "enqueuing into default, state 1 hop 1"); + ck_assert_str_eq(log.log_lines[0].type, "Job"); + ck_assert_str_eq(log.log_lines[0].date, "10/03/2016 13:43:35.194"); + ck_assert_str_eq(log.log_lines[0].obj, "PBS_Server.6728"); + i = log.ll_cur_amm-2; + ck_assert_str_eq(log.log_lines[i].msg, "Exit_status=143 resources_used.cput=1 resources_used.energy_used=0 resources_used.mem=51996kb resources_used.vmem=38060kb resources_used.walltime=00:00:42"); + ck_assert_str_eq(log.log_lines[i].type, "Job"); + ck_assert_str_eq(log.log_lines[i].date, "10/03/2016 13:44:35.174"); + ck_assert_str_eq(log.log_lines[i].obj, "PBS_Server.6172"); + + for (i = 0;i < log.ll_cur_amm;i++) + free_log_entry(&log.log_lines[i]); + free(log.log_lines); } END_TEST -START_TEST(test_two) +// test with gzip compressed log +START_TEST(test_20160928) { + struct tm *tm_ptr; + time_t t; + int file_count; + char *filenames[MAX_LOG_FILES_PER_DAY]; + const char *prefix_path = "."; + const char *jobid = "624[4]"; + gzFile fp; + struct log_array log = {}; + unsigned int i; + + // create tm struct + t = time(NULL); + tm_ptr = localtime(&t); + ck_assert(tm_ptr != NULL); + + // set to 2016-09-28 + tm_ptr->tm_year = 2016 - 1900; + tm_ptr->tm_mon = 9 - 1; + tm_ptr->tm_mday = 28; + + file_count = log_path((char *)prefix_path, 1, tm_ptr, filenames); + + ck_assert_int_eq(file_count, 2); + ck_assert_str_eq(filenames[0], "./server_logs/20160928"); + ck_assert_str_eq(filenames[1], "./server_logs/20160928.gz"); + + fp = gzopen(filenames[1], "r"); + ck_assert_msg(fp != NULL, "Failed to open ./server_logs/20160928.gz with gzopen"); + + ck_assert_int_eq(parse_log(&fp, (char *)jobid, 1, &log), 0); + + gzclose(fp); + free(filenames[0]); + free(filenames[1]); + + // check first and one to last line + ck_assert_str_eq(log.log_lines[0].msg, "enqueuing into short, state 2 hop 1"); + ck_assert_str_eq(log.log_lines[0].type, "Job"); + ck_assert_str_eq(log.log_lines[0].date, "09/28/2016 10:11:21.389"); + ck_assert_str_eq(log.log_lines[0].obj, "PBS_Server.6173"); + i = log.ll_cur_amm-2; + ck_assert_str_eq(log.log_lines[i].msg, "Exit_status=1 resources_used.cput=1 resources_used.energy_used=0 resources_used.mem=68752kb resources_used.vmem=38204kb resources_used.walltime=00:02:02"); + ck_assert_str_eq(log.log_lines[i].type, "Job"); + ck_assert_str_eq(log.log_lines[i].date, "09/28/2016 10:13:43.451"); + ck_assert_str_eq(log.log_lines[i].obj, "PBS_Server.6173"); + for (i = 0;i < log.ll_cur_amm;i++) + free_log_entry(&log.log_lines[i]); + free(log.log_lines); } END_TEST Suite *tracejob_suite(void) { Suite *s = suite_create("tracejob_suite methods"); - TCase *tc_core = tcase_create("test_one"); - tcase_add_test(tc_core, test_one); + TCase *tc_core = tcase_create("test_20161003"); + tcase_add_test(tc_core, test_20161003); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_20160928"); + tcase_add_test(tc_core, test_20160928); suite_add_tcase(s, tc_core); return s; From c590ce25cb3cef4239634e142dea48f7b8d32197 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Thu, 6 Oct 2016 13:47:16 +0200 Subject: [PATCH 039/312] Fix memory issue Pointer to static char cannot be freed. --- src/tools/tracejob.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tools/tracejob.c b/src/tools/tracejob.c index a861e38348..0e48ad0e80 100644 --- a/src/tools/tracejob.c +++ b/src/tools/tracejob.c @@ -419,7 +419,7 @@ int parse_log( int j = 0; struct tm tms; /* used to convert date to unix date */ - static char none[1] = { '\0' }; + const char none = '\0'; int lineno = 0; int logcount = 0; @@ -530,27 +530,27 @@ int parse_log( if (tmp.event != NULL) log->log_lines[log->ll_cur_amm].event = strdup(tmp.event); else - log->log_lines[log->ll_cur_amm].event = none; + log->log_lines[log->ll_cur_amm].event = strdup(&none); if (tmp.obj != NULL) log->log_lines[log->ll_cur_amm].obj = strdup(tmp.obj); else - log->log_lines[log->ll_cur_amm].obj = none; + log->log_lines[log->ll_cur_amm].obj = strdup(&none); if (tmp.type != NULL) log->log_lines[log->ll_cur_amm].type = strdup(tmp.type); else - log->log_lines[log->ll_cur_amm].type = none; + log->log_lines[log->ll_cur_amm].type = strdup(&none); if (tmp.name != NULL) log->log_lines[log->ll_cur_amm].name = strdup(tmp.name); else - log->log_lines[log->ll_cur_amm].name = none; + log->log_lines[log->ll_cur_amm].name = strdup(&none); if (tmp.msg != NULL) log->log_lines[log->ll_cur_amm].msg = strdup(tmp.msg); else - log->log_lines[log->ll_cur_amm].msg = none; + log->log_lines[log->ll_cur_amm].msg = strdup(&none); switch (ind) { From 282999b2e96cd1298f22c46e9e59185254229e90 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 19 Oct 2016 16:39:58 -0600 Subject: [PATCH 040/312] Fix a warning. --- src/tools/tracejob.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tools/tracejob.c b/src/tools/tracejob.c index 0e48ad0e80..d19dc6d3e1 100644 --- a/src/tools/tracejob.c +++ b/src/tools/tracejob.c @@ -121,7 +121,7 @@ int main( int event_type; char filter_excessive = 0; int excessive_count; - struct log_array log = {}; + struct log_array log; #if defined(FILTER_EXCESSIVE) filter_excessive = 1; @@ -131,6 +131,8 @@ int main( excessive_count = EXCESSIVE_COUNT; #endif + memset(&log, 0, sizeof(log)); + while ((c = getopt(argc, argv, "qzvamslw:p:n:f:c:")) != EOF) { switch (c) From d8348b6e7fcdafc33349b85adc6a6f0a6f292368 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Thu, 20 Oct 2016 14:14:50 +0200 Subject: [PATCH 041/312] Only execute ldconfig when run as root --- src/daemon_client/Makefile.am | 6 +++--- src/resmom/Makefile.am | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/daemon_client/Makefile.am b/src/daemon_client/Makefile.am index 57ca59bd18..064a9c954c 100644 --- a/src/daemon_client/Makefile.am +++ b/src/daemon_client/Makefile.am @@ -1,6 +1,6 @@ include $(top_srcdir)/buildutils/config.mk -noinst_HEADERS = trq_auth_daemon.h +noinst_HEADERS = trq_auth_daemon.h DIST_SUBDIRS = CLEANFILES = *.gcda *.gcno *.gcov @@ -30,7 +30,7 @@ install-exec-hook: if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - if [ $(FROM_MAKE_PACKAGES)0 -eq 0 ]; then \ + if [ $(FROM_MAKE_PACKAGES)0 -eq 0 && $(UID) -eq 0 ]; then \ /sbin/ldconfig; \ fi; \ fi; \ @@ -41,7 +41,7 @@ uninstall-hook: rm -f $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || :; \ rm -f $(DESTDIR)/etc/init.d/trqauthd || :; \ rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ - if [ $(FROM_MAKE_PACKAGES)0 -eq 0 ]; then \ + if [ $(FROM_MAKE_PACKAGES)0 -eq 0 && $(UID) -eq 0 ]; then \ /sbin/ldconfig; \ fi; \ fi diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index c6c9acb430..80725aba6e 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -48,7 +48,7 @@ pbs_mom_SOURCES = catch_child.c mom_comm.c mom_inter.c mom_main.c \ ../server/attr_recov.c ../server/dis_read.c \ ../server/job_attr_def.c ../server/job_recov.c \ ../server/reply_send.c ../server/resc_def_all.c \ - ../server/job_qs_upgrade.c + ../server/job_qs_upgrade.c if BUILDCPA pbs_mom_SOURCES += cray_cpa.c endif @@ -90,7 +90,7 @@ install-exec-hook: if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - if [ $(FROM_MAKE_PACKAGES)0 -eq 0 ]; then \ + if [ $(FROM_MAKE_PACKAGES)0 -eq 0 && $(UID) -eq 0 ]; then \ /sbin/ldconfig; \ fi; \ fi; \ @@ -103,7 +103,7 @@ uninstall-hook: rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service || :; \ rm -f $(DESTDIR)/etc/init.d/pbs_mom || :; \ rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ - if [ $(FROM_MAKE_PACKAGES)0 -eq 0 ]; then \ + if [ $(FROM_MAKE_PACKAGES)0 -eq 0 && $(UID) -eq 0 ]; then \ /sbin/ldconfig; \ fi; \ fi From 572dd20b5b47eb4e71543ea63e1b238f268cfd18 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 24 Oct 2016 13:56:22 -0600 Subject: [PATCH 042/312] TRQ-3770. Add two comments and change the wording on a message for the plugin driver. --- contrib/resource_plugin_kit/plugin_driver.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/resource_plugin_kit/plugin_driver.cpp b/contrib/resource_plugin_kit/plugin_driver.cpp index 9303e62af5..8b4b2dc645 100644 --- a/contrib/resource_plugin_kit/plugin_driver.cpp +++ b/contrib/resource_plugin_kit/plugin_driver.cpp @@ -40,12 +40,15 @@ void test_usage_information() if (max_pid_id == 0) max_pid_id = 32768; + // Decrement because we'll add one later to make sure we avoid process 0 max_pid_id--; struct timeval now; gettimeofday(&now, 0); srand(now.tv_usec); + // Get one random pid to test. Make changes here if you'd like to test with more than 1 + // pid, or if you'd like specific pids. do { random_pid = rand() % max_pid_id + 1; // add 1 to avoid process 0 @@ -54,7 +57,7 @@ void test_usage_information() job_pids.insert(random_pid); report_job_resources(jid, job_pids, usage_information); - printf("Your plugin reported the following for pid %d:\n", random_pid); + printf("Your plugin reported the following for the random pid %d:\n", random_pid); for (std::map::iterator it = usage_information.begin(); it != usage_information.end(); it++) From 4a1570239ab92251aa24dfcfb677b181b0261e73 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 25 Oct 2016 11:55:33 -0600 Subject: [PATCH 043/312] TRQ-3813 & TRQ-3750. Fix some memory leaks. --- src/include/pbs_nodes.h | 2 +- src/lib/Libnet/net_cache.c | 3 +++ src/server/node_manager.c | 17 +------------- src/server/node_manager.h | 2 -- src/server/pbsd_init.c | 3 --- src/server/pbsd_main.c | 1 - src/server/process_mom_update.c | 32 ++++++++++++++++++++++---- src/server/receive_mom_communication.c | 18 +++++++++++---- src/server/req_modify_node.c | 1 - 9 files changed, 46 insertions(+), 33 deletions(-) diff --git a/src/include/pbs_nodes.h b/src/include/pbs_nodes.h index b61b6b3e31..a2d386125a 100644 --- a/src/include/pbs_nodes.h +++ b/src/include/pbs_nodes.h @@ -630,9 +630,9 @@ void free_prop_list(struct prop*); void reinitialize_node_iterator(node_iterator *); int mgr_set_node_attr(struct pbsnode *, attribute_def *, int, svrattrl *, int, int *, void *, int, bool); -#ifdef BATCH_REQUEST_H void setup_notification(char *); +#ifdef BATCH_REQUEST_H struct pbsnode *find_nodebynameandaltname(char *, char *); void free_prop_attr(pbs_attribute*); void recompute_ntype_cnts(); diff --git a/src/lib/Libnet/net_cache.c b/src/lib/Libnet/net_cache.c index 32726d402c..820998d1e4 100644 --- a/src/lib/Libnet/net_cache.c +++ b/src/lib/Libnet/net_cache.c @@ -195,7 +195,10 @@ class addrcache { if ((pAddr->ai_family != AF_INET) || (cacheDestroyed == TRUE)) + { + freeaddrinfo(pAddr); return(NULL); + } struct addrinfo *pTmpAddr = NULL; char key[65]; diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 52bd9a21e6..6b488a6bdb 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -182,7 +182,6 @@ extern char *path_nodenote_new; extern char server_name[]; extern struct server server; -extern tlist_head svr_newnodes; extern attribute_def node_attr_def[]; /* node attributes defs */ extern int SvrNodeCt; @@ -1259,7 +1258,6 @@ void setup_notification( { struct pbsnode *pnode; - new_node *nnew; if (pname != NULL) { @@ -1268,20 +1266,6 @@ void setup_notification( /* call it offline until after all nodes get the new ipaddr */ pnode->nd_state |= INUSE_OFFLINE; - nnew = (new_node *)calloc(1, sizeof(new_node)); - - if (nnew == NULL) - { - pnode->unlock_node(__func__, "nnew == NULL", LOGLEVEL); - return; - } - - CLEAR_LINK(nnew->nn_link); - - nnew->nn_name = strdup(pname); - - append_link(&svr_newnodes, &nnew->nn_link, nnew); - pnode->unlock_node(__func__, "nnew != NULL", LOGLEVEL); } } @@ -5056,6 +5040,7 @@ int set_nodes( if ((rc = translate_howl_to_string(mic_list, EMsg, NCount, mic_str, NULL, false)) != PBSE_NONE) { free_nodes(pjob, spec); + delete [] ard_array; return(rc); } diff --git a/src/server/node_manager.h b/src/server/node_manager.h index e37aaa6080..14d27c6667 100644 --- a/src/server/node_manager.h +++ b/src/server/node_manager.h @@ -17,8 +17,6 @@ job *find_job_by_node(struct pbsnode *pnode, char *jobid); void update_job_data(struct pbsnode *np, char *jobstring_in); -void setup_notification(char *pname); - int is_stat_get(const char *node_name, struct tcp_chan *chan); int is_compose(struct tcp_chan *chan, int command); diff --git a/src/server/pbsd_init.c b/src/server/pbsd_init.c index 83bf507657..8727f2f46f 100644 --- a/src/server/pbsd_init.c +++ b/src/server/pbsd_init.c @@ -203,7 +203,6 @@ extern char *path_pbs_environment; extern int queue_rank; extern char server_name[]; -extern tlist_head svr_newnodes; extern std::list *task_list_timed; extern pthread_mutex_t task_list_timed_mutex; task_recycler tr; @@ -983,8 +982,6 @@ int initialize_data_structures_and_mutexes() initialize_task_recycler(); - CLEAR_HEAD(svr_newnodes); - if (cray_enabled == true) { initialize_login_holder(); diff --git a/src/server/pbsd_main.c b/src/server/pbsd_main.c index 9ca5590d2a..530abd7458 100644 --- a/src/server/pbsd_main.c +++ b/src/server/pbsd_main.c @@ -301,7 +301,6 @@ extern int listener_command; //extern hello_container hellos; //extern hello_container failures; pthread_mutex_t *listener_command_mutex; -tlist_head svr_newnodes; /* list of newly created nodes */ pthread_mutex_t task_list_timed_mutex; pid_t sid; diff --git a/src/server/process_mom_update.c b/src/server/process_mom_update.c index f43310a5b4..29e8ba5331 100644 --- a/src/server/process_mom_update.c +++ b/src/server/process_mom_update.c @@ -613,11 +613,17 @@ int process_uname_str( +/* + * process_state_str() + * + * Processes the state string that came from the mom + * @retun PBSE_NONE on SUCCESS, PBSE_* on an error + */ int process_state_str( - struct pbsnode *np, - const char *str) + pbsnode *np, + const char *str) { char log_buf[LOCAL_LOG_BUF_SIZE]; @@ -629,7 +635,7 @@ int process_state_str( np->get_name()); log_err(-1, __func__, log_buf); - return PBSE_HIERARCHY_NOT_SENT; + return(PBSE_HIERARCHY_NOT_SENT); } if (!strncmp(str, "state=down", 10)) { @@ -676,6 +682,8 @@ int process_state_str( return(rc); } /* END process_state_str() */ + + int update_node_mac_addr( struct pbsnode *np, const char *str) @@ -792,6 +800,14 @@ void update_layout_if_needed( +/* + * process_status_info() + * + * @param nd_name - the name of the node who sent this update + * @param status_info - a list of each status string sent in + * @return PBSE_NONE on SUCCESS, PBSE_* on error + */ + int process_status_info( const char *nd_name, @@ -799,7 +815,7 @@ int process_status_info( { const char *name = nd_name; - struct pbsnode *current; + pbsnode *current; bool mom_job_sync = true; bool auto_np = false; bool down_on_error = false; @@ -834,6 +850,7 @@ int process_status_info( for (unsigned int i = 0; i != status_info.size(); i++) { const char *str = status_info[i].c_str(); + /* these two options are for switching nodes */ if (!strncmp(str, NUMA_KEYWORD, strlen(NUMA_KEYWORD))) { @@ -1013,6 +1030,13 @@ int process_status_info( +/* + * move_past_gpu_status() + * + * @param i - the current index of which string we're processing in the list + * @param status_info - the list of status strings + */ + void move_past_gpu_status( unsigned int &i, diff --git a/src/server/receive_mom_communication.c b/src/server/receive_mom_communication.c index 9949edc4d0..69e2cc2e41 100644 --- a/src/server/receive_mom_communication.c +++ b/src/server/receive_mom_communication.c @@ -146,26 +146,34 @@ void get_status_info( +/* + * is_reporter_node() + * + * @param node_id - the id of the node that we're checking + */ -int is_reporter_node( +bool is_reporter_node( const char *node_id) { - struct pbsnode *pnode = find_nodebyname(node_id); - int rc = FALSE; + pbsnode *pnode = find_nodebyname(node_id); + bool is_reporter = false; if (pnode != NULL) { - rc = pnode->nd_is_alps_reporter; + is_reporter = (bool)pnode->nd_is_alps_reporter; pnode->unlock_node(__func__, NULL, LOGLEVEL); } - return(rc); + return(is_reporter); } /* END is_reporter_node() */ +/* + * is_stat_get() + */ int is_stat_get( diff --git a/src/server/req_modify_node.c b/src/server/req_modify_node.c index 84477dde7e..427fb36a5e 100644 --- a/src/server/req_modify_node.c +++ b/src/server/req_modify_node.c @@ -131,7 +131,6 @@ #include "work_task.h" #include "mcom.h" #include "node_func.h" /* init_prop, find_nodebyname, reinitialize_node_iterator, recompute_ntype_cnts, effective_node_delete, create_pbs_node */ -#include "node_manager.h" /* setup_notification */ #include "queue_func.h" /* find_queuebyname, que_alloc, que_free */ #include "queue_recov.h" /* que_save */ #include "mutex_mgr.hpp" From acd5d9afb7942aae964eedbd4f4036c00c721a8c Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 27 Oct 2016 00:04:25 +0000 Subject: [PATCH 044/312] TRQ-3801 - add locking to mail throttler methods --- src/include/mail_throttler.hpp | 2 ++ src/server/mail_throttler.cpp | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/include/mail_throttler.hpp b/src/include/mail_throttler.hpp index bfd5bf8bdf..3966ccb859 100644 --- a/src/include/mail_throttler.hpp +++ b/src/include/mail_throttler.hpp @@ -5,6 +5,7 @@ #include #include #include +#include class mail_info @@ -29,6 +30,7 @@ class mail_info class mail_throttler { std::map > pending; + pthread_mutex_t mt_mutex; public: mail_throttler(); diff --git a/src/server/mail_throttler.cpp b/src/server/mail_throttler.cpp index 461cbef89d..47f4aa787e 100644 --- a/src/server/mail_throttler.cpp +++ b/src/server/mail_throttler.cpp @@ -61,7 +61,12 @@ int mail_throttler::get_email_list( { int rc = PBSE_NONE; - std::map >::iterator it = this->pending.find(addressee); + std::map >::iterator it; + + // set lock + pthread_mutex_lock(&this->mt_mutex); + + it = this->pending.find(addressee); if (it != this->pending.end()) { @@ -75,6 +80,9 @@ int mail_throttler::get_email_list( list.clear(); } + // unlock + pthread_mutex_unlock(&this->mt_mutex); + return(rc); } // END get_email_list() @@ -94,7 +102,12 @@ bool mail_throttler::add_email_entry( { bool is_first = false; - std::map >::iterator it = this->pending.find(mi.mailto); + std::map >::iterator it; + + // set lock + pthread_mutex_lock(&this->mt_mutex); + + it = this->pending.find(mi.mailto); if (it == this->pending.end()) { @@ -111,6 +124,9 @@ bool mail_throttler::add_email_entry( is_first = this->pending[mi.mailto].size() == 1; } + // unlock + pthread_mutex_unlock(&this->mt_mutex); + return(is_first); } // END add_email_entry() From be00b2c70acc33f85f2700788e13340035828f7c Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 27 Oct 2016 17:15:22 -0600 Subject: [PATCH 045/312] TRQ-3750. Fix a few more memory leaks. --- src/lib/Libattr/attr_fn_resc.c | 8 +++++++ src/lib/Libattr/attr_str_conversion.c | 4 ++++ src/server/delete_all_tracker.cpp | 2 +- src/server/job_recycler.c | 1 + src/server/node_func.c | 4 ++-- src/server/req_stat.c | 32 +++++++++------------------ 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/lib/Libattr/attr_fn_resc.c b/src/lib/Libattr/attr_fn_resc.c index 7f4b6595e1..a8c54aaa0b 100644 --- a/src/lib/Libattr/attr_fn_resc.c +++ b/src/lib/Libattr/attr_fn_resc.c @@ -610,7 +610,15 @@ void free_resc( std::vector *resources = (std::vector *)pattr->at_val.at_ptr; if (resources != NULL) + { + for (size_t i = 0; i < resources->size(); i++) + { + resource &r = resources->at(i); + r.rs_defin->rs_free(&r.rs_value); + } + delete resources; + } pattr->at_val.at_ptr = NULL; pattr->at_flags &= ~ATR_VFLAG_SET; diff --git a/src/lib/Libattr/attr_str_conversion.c b/src/lib/Libattr/attr_str_conversion.c index fbad83f80d..7a755b4f4b 100644 --- a/src/lib/Libattr/attr_str_conversion.c +++ b/src/lib/Libattr/attr_str_conversion.c @@ -461,6 +461,10 @@ int str_to_attr( unescape_xml(val,buf,sizeof(buf)); + // Free the old value, if present + if (attr[index].at_val.at_str != NULL) + free(attr[index].at_val.at_str); + attr[index].at_val.at_str = strdup(buf); if (attr[index].at_val.at_str == NULL) diff --git a/src/server/delete_all_tracker.cpp b/src/server/delete_all_tracker.cpp index 167ca55e3e..b475ae9f2d 100644 --- a/src/server/delete_all_tracker.cpp +++ b/src/server/delete_all_tracker.cpp @@ -82,7 +82,7 @@ const char *mgr = "manager"; -delete_all_tracker::delete_all_tracker() +delete_all_tracker::delete_all_tracker() : qdel_map() { pthread_mutex_init(&this->lock, NULL); diff --git a/src/server/job_recycler.c b/src/server/job_recycler.c index a879ab0d23..c69a57c6e2 100644 --- a/src/server/job_recycler.c +++ b/src/server/job_recycler.c @@ -171,6 +171,7 @@ int insert_into_recycler( pthread_mutex_lock(recycler.rc_mutex); + // Make sure that we aren't inserting a job twice if (pjob->ji_being_recycled == true) { pthread_mutex_unlock(recycler.rc_mutex); diff --git a/src/server/node_func.c b/src/server/node_func.c index be3a035ee6..8d41c3d4ca 100644 --- a/src/server/node_func.c +++ b/src/server/node_func.c @@ -611,8 +611,6 @@ int status_nodeattrib( atemp[i].at_val.at_short = pnode->nd_power_state; else if (i == ND_ATR_properties) { - // NOTE: see below for nd_properties encoding - atemp[i].at_val.at_arst = pnode->nd_prop; } else if (i == ND_ATR_status) atemp[i].at_val.at_str = NULL; @@ -756,6 +754,8 @@ int status_nodeattrib( if ((index == ND_ATR_jobs) && (pnode->nd_is_alps_login == TRUE)) rc = login_encode_jobs(pnode, phead); + else if (index == ND_ATR_properties) + rc = pnode->encode_properties(phead); else { if (index == ND_ATR_status) diff --git a/src/server/req_stat.c b/src/server/req_stat.c index e397b1ff5a..e3282dab0b 100644 --- a/src/server/req_stat.c +++ b/src/server/req_stat.c @@ -190,7 +190,7 @@ int req_stat_job( struct batch_request *preq) /* ptr to the decoded request */ { - struct stat_cntl *cntl; /* see svrfunc.h */ + struct stat_cntl cntl; /* see svrfunc.h */ char *name; job *pjob = NULL; pbs_queue *pque = NULL; @@ -299,17 +299,6 @@ int req_stat_job( CLEAR_HEAD(preq->rq_reply.brp_un.brp_status); - cntl = (struct stat_cntl *)calloc(1, sizeof(struct stat_cntl)); - - if (cntl == NULL) - { - if (pque != NULL) - unlock_queue(pque, "req_stat_job", (char *)"no memory cntl", LOGLEVEL); - req_reject(PBSE_SYSTEM, 0, preq, NULL, NULL); - - return(PBSE_SYSTEM); - } - if ((type == tjstTruncatedQueue) || (type == tjstTruncatedServer)) { @@ -320,20 +309,21 @@ int req_stat_job( } } - cntl->sc_type = (int)type; - cntl->sc_conn = -1; - cntl->sc_pque = pque; - cntl->sc_origrq = preq; - cntl->sc_post = req_stat_job_step2; - cntl->sc_jobid[0] = '\0'; /* cause "start from beginning" */ - cntl->sc_condensed = condensed; + memset(&cntl, 0, sizeof(cntl)); + + cntl.sc_type = (int)type; + cntl.sc_conn = -1; + cntl.sc_pque = pque; + cntl.sc_origrq = preq; + cntl.sc_post = req_stat_job_step2; + cntl.sc_jobid[0] = '\0'; /* cause "start from beginning" */ + cntl.sc_condensed = condensed; - req_stat_job_step2(cntl); /* go to step 2, see if running is current */ + req_stat_job_step2(&cntl); /* go to step 2, see if running is current */ if (pque != NULL) unlock_queue(pque, "req_stat_job", (char *)"success", LOGLEVEL); - free(cntl); return(PBSE_NONE); } /* END req_stat_job() */ From f98e9077c2242e468647f70a46d5625675100c90 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 28 Oct 2016 13:42:21 -0600 Subject: [PATCH 046/312] Fix Makefile.am conditional bug in install/uninstall hook --- src/daemon_client/Makefile.am | 4 ++-- src/resmom/Makefile.am | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/daemon_client/Makefile.am b/src/daemon_client/Makefile.am index 064a9c954c..cf9c944d36 100644 --- a/src/daemon_client/Makefile.am +++ b/src/daemon_client/Makefile.am @@ -30,7 +30,7 @@ install-exec-hook: if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - if [ $(FROM_MAKE_PACKAGES)0 -eq 0 && $(UID) -eq 0 ]; then \ + if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test $(UID) -eq 0; then \ /sbin/ldconfig; \ fi; \ fi; \ @@ -41,7 +41,7 @@ uninstall-hook: rm -f $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || :; \ rm -f $(DESTDIR)/etc/init.d/trqauthd || :; \ rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ - if [ $(FROM_MAKE_PACKAGES)0 -eq 0 && $(UID) -eq 0 ]; then \ + if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test $(UID) -eq 0; then \ /sbin/ldconfig; \ fi; \ fi diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index 80725aba6e..e553d00f22 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -90,7 +90,7 @@ install-exec-hook: if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - if [ $(FROM_MAKE_PACKAGES)0 -eq 0 && $(UID) -eq 0 ]; then \ + if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test $(UID) -eq 0; then \ /sbin/ldconfig; \ fi; \ fi; \ @@ -103,7 +103,7 @@ uninstall-hook: rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service || :; \ rm -f $(DESTDIR)/etc/init.d/pbs_mom || :; \ rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ - if [ $(FROM_MAKE_PACKAGES)0 -eq 0 && $(UID) -eq 0 ]; then \ + if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test $(UID) -eq 0; then \ /sbin/ldconfig; \ fi; \ fi From 7a5cb345ae0ece71501604f6b25d27fefeb06f48 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 28 Oct 2016 15:24:33 -0600 Subject: [PATCH 047/312] TRQ-3829. Add a presetup_prologue that can execute before the mom does anything. This allows integration with kerberos. --- src/include/mom_config.h | 2 ++ src/include/mom_func.h | 13 ++++++----- src/resmom/parse_config.c | 32 +++++++++++++++++++++++++- src/resmom/prolog.c | 38 +++++++++++++++++++++++++++++-- src/resmom/start_exec.c | 25 ++++++++++++++++++++ src/test/start_exec/scaffolding.c | 1 + 6 files changed, 102 insertions(+), 9 deletions(-) diff --git a/src/include/mom_config.h b/src/include/mom_config.h index e76fbb58a7..5634cfd2c6 100644 --- a/src/include/mom_config.h +++ b/src/include/mom_config.h @@ -79,6 +79,7 @@ #include +#include #include "resmon.h" #include "pbs_ifl.h" @@ -172,6 +173,7 @@ extern int max_join_job_wait_time; extern int resend_join_job_wait_time; extern int mom_hierarchy_retry_time; extern int MOMJobDirStickySet; +extern std::string presetup_prologue; struct specials { diff --git a/src/include/mom_func.h b/src/include/mom_func.h index 7858e7a19d..6d3c8f09a0 100644 --- a/src/include/mom_func.h +++ b/src/include/mom_func.h @@ -281,12 +281,13 @@ extern int kill_task(job *,struct task *, int, int); #define PE_IO_TYPE_ASIS 0 #define PE_IO_TYPE_STD 1 -#define PE_PROLOG 1 -#define PE_EPILOG 2 -#define PE_PROLOGUSER 3 -#define PE_EPILOGUSER 4 -#define PE_PROLOGUSERJOB 5 /* per job prologue script */ -#define PE_EPILOGUSERJOB 6 /* per job epilogue script */ +#define PE_PROLOG 1 +#define PE_EPILOG 2 +#define PE_PROLOGUSER 3 +#define PE_EPILOGUSER 4 +#define PE_PROLOGUSERJOB 5 /* per job prologue script */ +#define PE_EPILOGUSERJOB 6 /* per job epilogue script */ +#define PE_PRESETUPPROLOG 7 /* prior to becoming the user */ #ifdef LIBPBS_H extern int open_std_file(job *, enum job_file, int, gid_t); diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index d7582a9a35..beaf5621a5 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -188,7 +188,7 @@ short memory_pressure_duration = 0; /* 0: off, >0: check and kill */ int max_join_job_wait_time = MAX_JOIN_WAIT_TIME; int resend_join_job_wait_time = RESEND_WAIT_TIME; int mom_hierarchy_retry_time = NODE_COMM_RETRY_TIME; - +std::string presetup_prologue; @@ -298,6 +298,7 @@ unsigned long setresendjoinjobwaittime(const char *); unsigned long setmomhierarchyretrytime(const char *); unsigned long setjobdirectorysticky(const char *); unsigned long setcudavisibledevices(const char *); +unsigned long set_presetup_prologue(const char *); struct specials special[] = { { "force_overwrite", setforceoverwrite}, @@ -382,6 +383,7 @@ struct specials special[] = { { "jobdirectory_sticky", setjobdirectorysticky}, { "cuda_visible_devices", setcudavisibledevices}, { "cray_check_rur", setrur }, + { "presetup_prologue", set_presetup_prologue}, { NULL, NULL } }; @@ -2734,6 +2736,34 @@ unsigned long setmomhierarchyretrytime( +unsigned long set_presetup_prologue( + + const char *value) + + { + log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, __func__, value); + + if (value != NULL) + { + presetup_prologue = value; + if (presetup_prologue[0] != '/') + { + sprintf(log_buffer, "$presetup_prologue must be given an absolute path, but received '%s'", + value); + log_err(-1, __func__, log_buffer); + + presetup_prologue.clear(); + + // FIXME: For some reason, these things use 0 for error. + return(0); + } + } + + return(1); + } // END set_presetup_prologue() + + + const char *arch( struct rm_attribute *attrib) /* I */ diff --git a/src/resmom/prolog.c b/src/resmom/prolog.c index 19c87e46ba..63a7424477 100644 --- a/src/resmom/prolog.c +++ b/src/resmom/prolog.c @@ -540,6 +540,32 @@ int check_pelog_permissions( +/* + * get_complete_hostlist() + * + */ + +char *get_complete_hostlist( + + job *pjob) + + { + std::string complete_hostlist; + std::string previous_host; + + for (int i = 0; i < pjob->ji_numnodes; i++) + { + if (i != 0) + complete_hostlist += ","; + + complete_hostlist += pjob->ji_hosts[i].hn_host; + } + + return(strdup(complete_hostlist.c_str())); + } // END get_complete_hostlist() + + + /* * setup_pelog_arguments() * @@ -612,13 +638,21 @@ void setup_pelog_arguments( else { /* prologue */ + int last_arg = 9; arg[5] = strdup(resc_to_string(pjob, JOB_ATR_resource, resc_list, sizeof(resc_list))); arg[6] = pjob->ji_wattr[JOB_ATR_in_queue].at_val.at_str; arg[7] = pjob->ji_wattr[JOB_ATR_account].at_val.at_str; arg[8] = strdup(namebuf); - arg[9] = NULL; - LastArg = 9; + if (which == PE_PRESETUPPROLOG) + { + arg[9] = get_complete_hostlist(pjob); + last_arg++; + } + + arg[last_arg] = NULL; + + LastArg = last_arg; } for (int aindex = 0;aindex < LastArg; aindex++) diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index 4a16410bd4..7ef76a7111 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -7185,6 +7185,26 @@ void recover_cpuset_reservation( +/* + * execute_presetup_prologue() + * + * Executes a prologue that happens prior to the job's setup + */ + +int execute_presetup_prologue( + + job *pjob) + + { + char presetup_path[MAXLINE * 2]; + + snprintf(presetup_path, sizeof(presetup_path), "%s", presetup_prologue.c_str()); + + return(run_pelog(PE_PRESETUPPROLOG, presetup_path, pjob, PE_IO_TYPE_NULL, FALSE)); + } // END execute_presetup_prologue() + + + /** * Start execution of a job. * @@ -7244,6 +7264,11 @@ int start_exec( if ((ret = job_nodes(*pjob)) != PBSE_NONE) return(ret); + if (presetup_prologue.size() > 0) + { + execute_presetup_prologue(pjob); + } + /* start_exec only executed on mother superior */ pjob->ji_nodeid = 0; /* I'm MS */ nodenum = pjob->ji_numnodes; diff --git a/src/test/start_exec/scaffolding.c b/src/test/start_exec/scaffolding.c index f25f535f70..a9ad650668 100644 --- a/src/test/start_exec/scaffolding.c +++ b/src/test/start_exec/scaffolding.c @@ -102,6 +102,7 @@ int ac_read_amount; int ac_errno; int job_saved; int task_saved; +std::string presetup_prologue; #ifdef NUMA_SUPPORT nodeboard node_boards[MAX_NODE_BOARDS]; From 48af0abf48ff1d492c0617a7c3b8cea203190985 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 2 Nov 2016 15:06:13 -0600 Subject: [PATCH 048/312] TRQ-3822. Add the ability to set a default gpu mode as a server parameter. This will help people migrate to CUDA 8. Our current code sets exclusive_thread as the default. In a future version, we'll change the default to exclusive_process, which is supported in CUDA 8. --- src/include/pbs_ifl.h | 5 ++- src/include/qmgr_svr_public.h | 1 + src/include/server.h | 1 + src/resmom/nvidia.c | 48 ++++++++++++++++++---- src/server/node_manager.c | 12 ++++-- src/server/req_manager.c | 62 +++++++++++++++++++++++++++++ src/server/svr_attr_def.c | 14 +++++++ src/test/job_recov/scaffolding.c | 5 +++ src/test/node_manager/scaffolding.c | 5 +++ src/test/pbsd_init/scaffolding.c | 2 +- src/test/req_manager/scaffolding.c | 1 + src/test/req_manager/test_uut.c | 39 ++++++++++++++++-- 12 files changed, 178 insertions(+), 17 deletions(-) diff --git a/src/include/pbs_ifl.h b/src/include/pbs_ifl.h index 2beaf14f31..f10eed1e18 100644 --- a/src/include/pbs_ifl.h +++ b/src/include/pbs_ifl.h @@ -378,10 +378,11 @@ #define ATTR_NODE_acl "acl" #define ATTR_NODE_requestid "requestid" -#define ATTR_cpustr "cpuset_string" -#define ATTR_memstr "memset_string" +#define ATTR_cpustr "cpuset_string" +#define ATTR_memstr "memset_string" #define ATTR_user_kill_delay "user_kill_delay" #define ATTR_idle_slot_limit "idle_slot_limit" +#define ATTR_default_gpu_mode "default_gpu_mode" #define ATTR_copy_on_rerun "copy_on_rerun" #define ATTR_job_exclusive_on_use "job_exclusive_on_use" #define ATTR_disable_automatic_requeue "disable_automatic_requeue" diff --git a/src/include/qmgr_svr_public.h b/src/include/qmgr_svr_public.h index a5dacb6f95..ba2b95732b 100644 --- a/src/include/qmgr_svr_public.h +++ b/src/include/qmgr_svr_public.h @@ -203,3 +203,4 @@ ATTR_tcpincomingtimeout, ATTR_ghost_array_recovery, ATTR_cgroup_per_task, ATTR_idle_slot_limit, +ATTR_default_gpu_mode, diff --git a/src/include/server.h b/src/include/server.h index 161504e727..9f3215bab1 100644 --- a/src/include/server.h +++ b/src/include/server.h @@ -228,6 +228,7 @@ enum srv_atr SRV_ATR_GhostArrayRecovery, SRV_ATR_CgroupPerTask, SRV_ATR_IdleSlotLimit, + SRV_ATR_DefaultGpuMode, /* This must be last */ SRV_ATR_LAST diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index 719efcf788..b4a3285ac6 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -1388,7 +1388,11 @@ void generate_server_gpustatus_nvml( snprintf(tmpbuf, 20, "gpu_fan_speed=%d%%", tmpint); gpu_status.push_back(tmpbuf); } - else + else if (rc != NVML_ERROR_NOT_SUPPORTED) + { + log_nvml_error (rc, NULL, __func__); + } + else if (LOGLEVEL >= 6) { log_nvml_error (rc, NULL, __func__); } @@ -1404,7 +1408,11 @@ void generate_server_gpustatus_nvml( snprintf(tmpbuf, 50, "gpu_memory_used=%lld MB", (mem_info.used/(1024*1024))); gpu_status.push_back(tmpbuf); } - else + else if (rc != NVML_ERROR_NOT_SUPPORTED) + { + log_nvml_error (rc, NULL, __func__); + } + else if (LOGLEVEL >= 6) { log_nvml_error (rc, NULL, __func__); } @@ -1450,7 +1458,11 @@ void generate_server_gpustatus_nvml( } gpu_status.push_back(s); } - else + else if (rc != NVML_ERROR_NOT_SUPPORTED) + { + log_nvml_error (rc, NULL, __func__); + } + else if (LOGLEVEL >= 6) { log_nvml_error (rc, NULL, __func__); } @@ -1467,7 +1479,11 @@ void generate_server_gpustatus_nvml( snprintf(tmpbuf, 100, "gpu_memory_utilization=%d%%", util_info.memory); gpu_status.push_back(tmpbuf); } - else + else if (rc != NVML_ERROR_NOT_SUPPORTED) + { + log_nvml_error (rc, NULL, __func__); + } + else if (LOGLEVEL >= 6) { log_nvml_error (rc, NULL, __func__); } @@ -1482,7 +1498,11 @@ void generate_server_gpustatus_nvml( (ecc_mode == NVML_FEATURE_ENABLED) ? "Enabled" : "Disabled"); gpu_status.push_back(tmpbuf); } - else + else if (rc != NVML_ERROR_NOT_SUPPORTED) + { + log_nvml_error (rc, NULL, __func__); + } + else if (LOGLEVEL >= 6) { log_nvml_error (rc, NULL, __func__); } @@ -1497,7 +1517,11 @@ void generate_server_gpustatus_nvml( snprintf(tmpbuf, 100, "gpu_single_bit_ecc_errors=%lld", ecc_counts); gpu_status.push_back(tmpbuf); } - else + else if (rc != NVML_ERROR_NOT_SUPPORTED) + { + log_nvml_error (rc, NULL, __func__); + } + else if (LOGLEVEL >= 6) { log_nvml_error (rc, NULL, __func__); } @@ -1512,7 +1536,11 @@ void generate_server_gpustatus_nvml( snprintf(tmpbuf, 100, "gpu_double_bit_ecc_errors=%lld", ecc_counts); gpu_status.push_back(tmpbuf); } - else + else if (rc != NVML_ERROR_NOT_SUPPORTED) + { + log_nvml_error (rc, NULL, __func__); + } + else if (LOGLEVEL >= 6) { log_nvml_error (rc, NULL, __func__); } @@ -1526,7 +1554,11 @@ void generate_server_gpustatus_nvml( snprintf(tmpbuf, 25, "gpu_temperature=%d C", tmpint); gpu_status.push_back(tmpbuf); } - else + else if (rc != NVML_ERROR_NOT_SUPPORTED) + { + log_nvml_error (rc, NULL, __func__); + } + else if (LOGLEVEL >= 6) { log_nvml_error (rc, NULL, __func__); } diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 6b488a6bdb..d79512141f 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -188,6 +188,7 @@ extern int SvrNodeCt; extern int multi_mom; const int network_fail_wait_time = 300; +int default_gpu_mode = -1; #define SKIP_NONE 0 #define SKIP_EXCLUSIVE 1 @@ -2188,9 +2189,15 @@ int proplist( have_gpus = TRUE; gpu_err_reset = FALSE; /* default to no */ - /* default value if no other gets specified */ + // default value if no other gets specified + char *deflt_gpu_mode = NULL; - gpu_mode_rqstd = gpu_exclusive_thread; + get_svr_attr_str(SRV_ATR_DefaultGpuMode, &deflt_gpu_mode); + + if (deflt_gpu_mode != NULL) + gpu_mode_rqstd = default_gpu_mode; + else + gpu_mode_rqstd = gpu_exclusive_thread; } else { @@ -2255,7 +2262,6 @@ int proplist( - /* ** Add the "global" spec to every sub-spec in "spec". ** RETURNS: allocated string buffer (must be freed externally) diff --git a/src/server/req_manager.c b/src/server/req_manager.c index 0fbd99ac78..505f515e94 100644 --- a/src/server/req_manager.c +++ b/src/server/req_manager.c @@ -161,6 +161,7 @@ extern char *msg_man_del; extern char *msg_man_set; extern char *msg_man_uns; extern time_t pbs_incoming_tcp_timeout; +extern int default_gpu_mode; //extern mom_hierarchy_t *mh; @@ -2692,6 +2693,67 @@ int extra_resc_chk( return(TRUE); } + + +/* + * check_default_gpu_mode_str() + * + * Makes sure the default is a valid gpu mode + * @return PBSE_NONE on success of PBSE_ATTRTYPE if a bad value + */ + +int check_default_gpu_mode_str( + + pbs_attribute *pattr, + void *pobj, + int actmode) + + { + int rc = PBSE_NONE; + char *gpu_mode = pattr->at_val.at_str; + + switch (actmode) + { + case ATR_ACTION_ALTER: + + // this shouldn't happen + if (gpu_mode == NULL) + { + return(PBSE_ATTRTYPE); + } + + // Make sure it's one of the acceptable gpu modes + if (!strcmp(gpu_mode, "exclusive_thread")) + { + default_gpu_mode = gpu_exclusive_thread; + } + else if (!strcmp(gpu_mode, "exclusive")) + { + default_gpu_mode = gpu_exclusive; + } + else if (!strcmp(gpu_mode, "exclusive_process")) + { + default_gpu_mode = gpu_exclusive_process; + } + else if (!strcmp(gpu_mode, "default")) + { + default_gpu_mode = gpu_normal; + } + else if (!strcmp(gpu_mode, "shared")) + { + default_gpu_mode = gpu_normal; + } + else + rc = PBSE_ATTRTYPE; + + break; + } + + return(rc); + } // END check_default_gpu_mode_str() + + + /* * free_extraresc() makes sure that the init_resc_defs() is called after * the list has changed by 'unset'. diff --git a/src/server/svr_attr_def.c b/src/server/svr_attr_def.c index a4db4a4f65..e42cf65cf6 100644 --- a/src/server/svr_attr_def.c +++ b/src/server/svr_attr_def.c @@ -119,6 +119,7 @@ extern void restore_attr_default (struct pbs_attribute *); int update_user_acls(pbs_attribute *pattr, void *pobject, int actmode); int update_group_acls(pbs_attribute *pattr, void *pobject, int actmode); int node_exception_check(pbs_attribute *pattr, void *pobject, int actmode); +int check_default_gpu_mode_str(pbs_attribute *pattr, void *pobject, int actmode); extern int keep_completed_val_check(pbs_attribute *pattr,void *pobj,int actmode); /* DIAGTODO: write diag_attr_def.c */ @@ -1637,4 +1638,17 @@ attribute_def svr_attr_def[] = PARENT_TYPE_SERVER }, + // SRV_ATR_DefaultGpuMode + {(char *)ATTR_default_gpu_mode, // "default_gpu_mode" + decode_str, + encode_str, + set_str, + comp_str, + free_null, + check_default_gpu_mode_str, + MGR_ONLY_SET, + ATR_TYPE_STR, + PARENT_TYPE_SERVER + }, + }; diff --git a/src/test/job_recov/scaffolding.c b/src/test/job_recov/scaffolding.c index 0001591dcb..27eadb63a7 100644 --- a/src/test/job_recov/scaffolding.c +++ b/src/test/job_recov/scaffolding.c @@ -529,3 +529,8 @@ bool job_array::is_deleted() const { return(this->being_deleted); } + +int check_default_gpu_mode_str(pbs_attribute *pattr, void *pobj, int mode) + { + return(PBSE_NONE); + } diff --git a/src/test/node_manager/scaffolding.c b/src/test/node_manager/scaffolding.c index f7d3675ddd..508eb765d7 100644 --- a/src/test/node_manager/scaffolding.c +++ b/src/test/node_manager/scaffolding.c @@ -450,6 +450,11 @@ int get_svr_attr_b(int index, bool *b) return(0); } +int get_svr_attr_str(int index, char **str_ptr) + { + return(0); + } + int process_alps_status( char *nd_name, diff --git a/src/test/pbsd_init/scaffolding.c b/src/test/pbsd_init/scaffolding.c index 7cc14a980b..96e5ebce36 100644 --- a/src/test/pbsd_init/scaffolding.c +++ b/src/test/pbsd_init/scaffolding.c @@ -92,7 +92,7 @@ const char *msg_init_unkstate = "Unable to recover job in strange substate: %d"; all_jobs array_summary; attribute_def svr_attr_def[10]; int a_opt_init = -1; -all_tasks task_list_timed; +std::list *task_list_timed; pthread_mutex_t task_list_timed_mutex; char *path_jobinfo_log; int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ diff --git a/src/test/req_manager/scaffolding.c b/src/test/req_manager/scaffolding.c index cfc40ad4e3..36595e05e6 100644 --- a/src/test/req_manager/scaffolding.c +++ b/src/test/req_manager/scaffolding.c @@ -39,6 +39,7 @@ attribute_def svr_attr_def[10]; int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ int svr_chngNodesfile = 0; int svr_totnodes = 0; +int default_gpu_mode; bool exit_called = false; time_t pbs_incoming_tcp_timeout; diff --git a/src/test/req_manager/test_uut.c b/src/test/req_manager/test_uut.c index 80ce90cb5c..7a44842be4 100644 --- a/src/test/req_manager/test_uut.c +++ b/src/test/req_manager/test_uut.c @@ -4,10 +4,43 @@ #include #include #include "pbs_error.h" -START_TEST(test_one) +#include "pbs_nodes.h" +#include "attribute.h" + +int check_default_gpu_mode_str(pbs_attribute *pattr, void *pobj, int actmode); +extern int default_gpu_mode; + +START_TEST(test_check_default_gpu_mode_str) { + pbs_attribute pattr; + pattr.at_flags |= ATR_VFLAG_SET; + + // Valid + pattr.at_val.at_str = strdup("exclusive_process"); + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); + fail_unless(default_gpu_mode == gpu_exclusive_process); + pattr.at_val.at_str = strdup("exclusive_thread"); + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); + fail_unless(default_gpu_mode == gpu_exclusive_thread); + pattr.at_val.at_str = strdup("exclusive"); + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); + fail_unless(default_gpu_mode == gpu_exclusive); + pattr.at_val.at_str = strdup("default"); + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); + fail_unless(default_gpu_mode == gpu_normal); + pattr.at_val.at_str = strdup("shared"); + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); + fail_unless(default_gpu_mode == gpu_normal); + // Spelling error + pattr.at_val.at_str = strdup("sharedf"); + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_ATTRTYPE); + // Other operations shouldn't cause an error + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_NOOP) == PBSE_NONE); + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_NEW) == PBSE_NONE); + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_RECOV) == PBSE_NONE); + fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_FREE) == PBSE_NONE); } END_TEST @@ -21,8 +54,8 @@ END_TEST Suite *req_manager_suite(void) { Suite *s = suite_create("req_manager_suite methods"); - TCase *tc_core = tcase_create("test_one"); - tcase_add_test(tc_core, test_one); + TCase *tc_core = tcase_create("test_check_default_gpu_mode_str"); + tcase_add_test(tc_core, test_check_default_gpu_mode_str); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_two"); From 28df70b747ba56b0c7d0275d4020a873bff84133 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 4 Nov 2016 10:40:18 -0600 Subject: [PATCH 049/312] Fix uid lookup in makefiles. --- src/daemon_client/Makefile.am | 4 ++-- src/resmom/Makefile.am | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/daemon_client/Makefile.am b/src/daemon_client/Makefile.am index cf9c944d36..132053f60e 100644 --- a/src/daemon_client/Makefile.am +++ b/src/daemon_client/Makefile.am @@ -30,7 +30,7 @@ install-exec-hook: if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test $(UID) -eq 0; then \ + if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test `id -u` -eq 0; then \ /sbin/ldconfig; \ fi; \ fi; \ @@ -41,7 +41,7 @@ uninstall-hook: rm -f $(DESTDIR)/usr/lib/systemd/system/trqauthd.service || :; \ rm -f $(DESTDIR)/etc/init.d/trqauthd || :; \ rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ - if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test $(UID) -eq 0; then \ + if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test `id -u` -eq 0; then \ /sbin/ldconfig; \ fi; \ fi diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index e553d00f22..70d7492bb0 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -90,7 +90,7 @@ install-exec-hook: if [ ! -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf ]; then \ echo $(libdir) > $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ chmod 644 $(DESTDIR)/etc/ld.so.conf.d/torque.conf; \ - if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test $(UID) -eq 0; then \ + if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test `id -u` -eq 0; then \ /sbin/ldconfig; \ fi; \ fi; \ @@ -103,7 +103,7 @@ uninstall-hook: rm -f $(DESTDIR)/usr/lib/systemd/system/pbs_mom.service || :; \ rm -f $(DESTDIR)/etc/init.d/pbs_mom || :; \ rm -f $(DESTDIR)/etc/ld.so.conf.d/torque.conf || :; \ - if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test $(UID) -eq 0; then \ + if test $(FROM_MAKE_PACKAGES)0 -eq 0 && test `id -u` -eq 0; then \ /sbin/ldconfig; \ fi; \ fi From 8ab26691bbdd82e19e103b1650e16664d9229360 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 9 Nov 2016 11:01:58 -0700 Subject: [PATCH 050/312] TRQ-3841. Fix two instances where we were unlocking an unlocked mutex. --- src/server/pbsd_init.c | 6 +++--- src/server/svr_jobfunc.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/pbsd_init.c b/src/server/pbsd_init.c index 8727f2f46f..db25179df8 100644 --- a/src/server/pbsd_init.c +++ b/src/server/pbsd_init.c @@ -2769,7 +2769,6 @@ void check_jobs_queue( { std::string queue_name(pjob->ji_qs.ji_queue); - std::string job_id(pjob->ji_qs.ji_jobid); unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL); @@ -2794,8 +2793,9 @@ void check_jobs_queue( } unlock_queue(pque, __func__, NULL, LOGLEVEL); - - pjob = svr_find_job(job_id.c_str(), TRUE); + + // The job hasn't been queued, so we just re-lock it this way + lock_ji_mutex(pjob, __func__, "1", LOGLEVEL); } // END check_jobs_queue() diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index 52a9e95c94..644d1f3e37 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -320,7 +320,7 @@ int insert_into_alljobs_by_rank( while ((pjcur = iter->get_next_item()) != NULL) { - mutex_mgr pjcur_mgr(pjcur->ji_mutex, true); + mutex_mgr pjcur_mgr(pjcur->ji_mutex, false); if (job_qrank > pjcur->ji_wattr[JOB_ATR_qrank].at_val.at_long) { pjcur_mgr.set_unlock_on_exit(false); From 384badb97c6c4077e2a4a07e185edc69f599979b Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 9 Nov 2016 12:05:16 -0700 Subject: [PATCH 051/312] TRQ-3810. Fix a scenario where we free a member of an unlocked job. This results in a crash depending on timing. --- src/server/req_signal.c | 13 ++++-- src/test/req_signal/scaffolding.c | 32 ++++++++++--- src/test/req_signal/test_uut.c | 78 +++++++++++++++++++++++++++++-- 3 files changed, 111 insertions(+), 12 deletions(-) diff --git a/src/server/req_signal.c b/src/server/req_signal.c index 99fa72aab8..755a4e54d2 100644 --- a/src/server/req_signal.c +++ b/src/server/req_signal.c @@ -252,6 +252,8 @@ int req_signaljob( return(PBSE_NONE); } /* END req_signaljob() */ + + /* * issue_signal - send an internally generated signal to a running job */ @@ -289,7 +291,8 @@ int issue_signal( strcpy(jobid, pjob->ji_qs.ji_jobid); strcpy(newreq->rq_ind.rq_signal.rq_jid, pjob->ji_qs.ji_jobid); - snprintf(newreq->rq_ind.rq_signal.rq_signame, sizeof(newreq->rq_ind.rq_signal.rq_signame), "%s", signame); + snprintf(newreq->rq_ind.rq_signal.rq_signame, sizeof(newreq->rq_ind.rq_signal.rq_signame), + "%s", signame); /* The newreq is freed in relay_to_mom (failure) * or in issue_Drequest (success) */ @@ -302,14 +305,14 @@ int issue_signal( unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); func(newreq); - *pjob_ptr = svr_find_job((char *)jobid, TRUE); + *pjob_ptr = svr_find_job(jobid, TRUE); } else if ((extend != NULL) && (!strcmp(extend, RERUNFORCE))) { if (pjob == NULL) { - *pjob_ptr = svr_find_job((char *)jobid, TRUE); + *pjob_ptr = svr_find_job(jobid, TRUE); pjob = *pjob_ptr; } /* The job state is normally set when the obit arrives. But since the @@ -336,6 +339,7 @@ int issue_signal( pjob->ji_momhandle = -1; unlock_ji_mutex(pjob, __func__, "8", LOGLEVEL); + *pjob_ptr = NULL; return(PBSE_SYSTEM); } @@ -353,6 +357,9 @@ int issue_signal( func(newreq); rc = PBSE_NONE; + + // We need to re-acquire the lock or set the pointer to NULL + *pjob_ptr = svr_find_job(jobid, TRUE); } else rc = PBSE_JOBNOTFOUND; diff --git a/src/test/req_signal/scaffolding.c b/src/test/req_signal/scaffolding.c index 41f8b053b1..a642f3ef77 100644 --- a/src/test/req_signal/scaffolding.c +++ b/src/test/req_signal/scaffolding.c @@ -7,16 +7,21 @@ #include "node_func.h" /* node_info */ #include "attribute.h" +bool find_job = true; +bool nullify_pointer = false; +int unlocked_job = 0; +int found_job = 0; +int relay_rc = 0; attribute_def job_attr_def[10]; int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ char scaff_buffer[1024]; -struct batch_request *alloc_br(int type) +batch_request *alloc_br(int type) { - fprintf(stderr, "The call to alloc_br to be mocked!!\n"); - exit(1); + static batch_request preq; + return(&preq); } void set_old_nodes(job *pjob) @@ -63,10 +68,20 @@ char *pbse_to_txt(int err) job *svr_find_job(const char *jobid, int get_subjob) { - fprintf(stderr, "The call to find_job to be mocked!!\n"); - exit(1); + static job pjob; + + if (find_job) + { + found_job++; + strcpy(pjob.ji_qs.ji_jobid, jobid); + return(&pjob); + } + + return(NULL); } + + job *chk_job_request(char *jobid, struct batch_request *preq) { //We're mocking so that the following field has the address of a job struct. @@ -98,6 +113,8 @@ int get_batch_request_id( int unlock_ji_mutex(job *pjob, const char *id, const char *msg, int logging) { + unlocked_job++; + return(0); } @@ -111,7 +128,10 @@ void log_record(int eventtype, int objclass, const char *objname, const char *te int relay_to_mom(job **pjob_ptr, batch_request *request, void (*func)(struct work_task *)) { - return(0); + if (nullify_pointer) + *pjob_ptr = NULL; + + return(relay_rc); } batch_request *duplicate_request(batch_request *preq, int type) diff --git a/src/test/req_signal/test_uut.c b/src/test/req_signal/test_uut.c index de60063f75..145db42271 100644 --- a/src/test/req_signal/test_uut.c +++ b/src/test/req_signal/test_uut.c @@ -4,7 +4,17 @@ #include #include #include "pbs_error.h" +#include "batch_request.h" +#include "attribute.h" + +int issue_signal(job **, const char *, void (*func)(batch_request *), void *extra, char *extend); + extern char scaff_buffer[]; +extern int unlocked_job; +extern int found_job; +extern int relay_rc; +extern bool nullify_pointer; +extern bool find_job; START_TEST(test_req_signaljob_relaying_msg) { @@ -22,13 +32,72 @@ START_TEST(test_req_signaljob_relaying_msg) } END_TEST -START_TEST(test_two) +void free_null(pbs_attribute *attr); +void dummy_func(batch_request *preq) {} + +void init_attr_def() + + { + extern attribute_def job_attr_def[]; + + for (int i = 0; i < 10; i++) + job_attr_def[i].at_free = free_null; + } + + +START_TEST(test_issue_signal) { + job pjob; + strcpy(pjob.ji_qs.ji_jobid, "1.nalthis"); + job *ptr = &pjob; + found_job = 0; + unlocked_job = 0; + relay_rc = 1; + + // Make sure that if we have a non-NULL job pointer the mutex is locked + // found_job is set each time it's 'locked' in svr_find_job and unlocked_job is set each + // time it's 'unlocked' in unlock_ji_mutex() + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, strdup("force")) == PBSE_NONE); + fail_unless(ptr != NULL); + fail_unless(found_job == unlocked_job); + + relay_rc = 0; + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, strdup("force")) == PBSE_NONE); + fail_unless(ptr != NULL); + fail_unless(found_job == unlocked_job); + + nullify_pointer = true; + relay_rc = 1; + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, strdup("force")) == PBSE_NONE); + fail_unless(ptr != NULL); + // On this code path we should have locked twice and unlocked only once, because we 'unlocked' + // in relay_to_mom() + fail_unless(found_job == unlocked_job + 1); + + found_job = 0; + unlocked_job = 0; + nullify_pointer = true; + relay_rc = 1; + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, NULL) != PBSE_NONE); + fail_unless(ptr == NULL); + // We come back NULL from relay_to_mom(), so we shouldn't change these variables + fail_unless(unlocked_job == 0); + fail_unless(found_job == 0); + + ptr = &pjob; + nullify_pointer = false; + relay_rc = 1; + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, NULL) != PBSE_NONE); + fail_unless(ptr != NULL); + fail_unless(unlocked_job == 0); + fail_unless(found_job == 0); + } END_TEST + Suite *req_signal_suite(void) { Suite *s = suite_create("req_signal_suite methods"); @@ -36,8 +105,8 @@ Suite *req_signal_suite(void) tcase_add_test(tc_core, test_req_signaljob_relaying_msg); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_issue_signal"); + tcase_add_test(tc_core, test_issue_signal); suite_add_tcase(s, tc_core); return s; @@ -51,6 +120,9 @@ int main(void) { int number_failed = 0; SRunner *sr = NULL; + + init_attr_def(); + rundebug(); sr = srunner_create(req_signal_suite()); srunner_set_log(sr, "req_signal_suite.log"); From bea653c3d2de06c7ec117dcf5b1994057b216f7b Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 9 Nov 2016 14:07:14 -0700 Subject: [PATCH 052/312] TRQ-3833. Fix an issue with removing dependencies before checking if the user has permissions to delete the job. --- src/server/req_delete.c | 17 ++++++++++------- src/server/req_register.c | 18 +++++++++++------- src/test/req_delete/scaffolding.c | 2 +- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/server/req_delete.c b/src/server/req_delete.c index 7c97b152a4..93d5acee81 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -171,7 +171,7 @@ extern int svr_chk_owner(struct batch_request *, job *); void chk_job_req_permissions(job **,struct batch_request *); void on_job_exit_task(struct work_task *); void remove_stagein(job **pjob_ptr); -extern void removeBeforeAnyDependencies(const char *pJobID); +extern void removeBeforeAnyDependencies(job **pjob_ptr); @@ -396,8 +396,6 @@ void force_purge_work( - - void ensure_deleted( struct work_task *ptask) /* I */ @@ -436,7 +434,6 @@ void ensure_deleted( - void setup_apply_job_delete_nanny( job *pjob, /* I */ @@ -490,6 +487,15 @@ int execute_job_delete( return(-1); } + removeBeforeAnyDependencies(&pjob); + + // The job disappeared unexpectedly + if (pjob == NULL) + { + reply_ack(preq); + return(PBSE_NONE); + } + mutex_mgr job_mutex(pjob->ji_mutex, true); if (LOGLEVEL >= 10) @@ -1166,10 +1172,7 @@ int handle_single_delete( } else { - std::string jobID = pjob->ji_qs.ji_jobid; unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); - removeBeforeAnyDependencies(jobID.c_str()); - /* send the asynchronous reply if needed */ if (preq_tmp != NULL) diff --git a/src/server/req_register.c b/src/server/req_register.c index c688475a03..6dfd840f1f 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -3384,16 +3384,17 @@ void removeAfterAnyDependency( /* * removeBeforeAnyDependencies() - * - * + * Forcibly removes before any dependencies from this job + * @param pjob_ptr - a pointer to the job's pointer */ void removeBeforeAnyDependencies( - const char *pJId) + job **pjob_ptr) { - job *pLockedJob = svr_find_job(pJId,FALSE); + job *pLockedJob = *pjob_ptr; + std::string jobid(pLockedJob->ji_qs.ji_jobid); if (pLockedJob != NULL) { @@ -3410,10 +3411,13 @@ void removeBeforeAnyDependencies( depend_job *pDepJob = pDep->dp_jobs[i]; std::string depID(pDepJob->dc_child); job_mutex.unlock(); - removeAfterAnyDependency(depID.c_str(),pJId); - pLockedJob = svr_find_job(pJId,FALSE); + removeAfterAnyDependency(depID.c_str(), jobid.c_str()); + pLockedJob = svr_find_job(jobid.c_str(), FALSE); if (pLockedJob == NULL) - return; + { + *pjob_ptr = NULL; + break; + } job_mutex.mark_as_locked(); } diff --git a/src/test/req_delete/scaffolding.c b/src/test/req_delete/scaffolding.c index d6bf832b7a..b4ef218c19 100644 --- a/src/test/req_delete/scaffolding.c +++ b/src/test/req_delete/scaffolding.c @@ -262,7 +262,7 @@ int relay_to_mom(job **pjob_ptr, batch_request *request, void (*func)(struct w return(bad_relay); } -void removeBeforeAnyDependencies(const char *) {} +void removeBeforeAnyDependencies(job **pjob_ptr) {} /* From 2b24d447ea03cd2c435c64de6d91bffdefcd968c Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 9 Nov 2016 14:32:05 -0700 Subject: [PATCH 053/312] TRQ-3843. Fix a compilation error if cgroups and geometry requeuts or cpusets are enabled. --- src/resmom/mom_comm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index fb61d6a48b..dc4ab79b6e 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -8307,7 +8307,7 @@ static int adoptSession( char jobid_copy[PBS_MAXSVRJOBID+1]; int other_id_len; -#ifdef PENABLE_LINUX26_CPUSETS +#if !defined(PENABLE_LINUX_CGROUPS) and defined(PENABLE_LINUX26_CPUSETS) unsigned int len; FILE *fp; From 9b0b394a8a8664541ad1888e9590494d93218193 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 14 Nov 2016 10:53:38 -0700 Subject: [PATCH 054/312] Make batch_request a class instead of a struct. --- src/include/attribute.h | 1 - src/include/batch_request.h | 27 +- src/include/pbs_job.h | 3 +- src/include/work_task.h | 2 +- src/lib/Libattr/attr_func.c | 1 - src/lib/Libifl/dec_svrattrl.c | 5 +- src/lib/Libnet/net_common.c | 9 +- src/resmom/catch_child.c | 4 +- src/resmom/mom_process_request.c | 118 ++--- src/resmom/mom_process_request.h | 2 - src/resmom/requests.c | 4 +- src/server/array_func.c | 6 +- src/server/batch_request.c | 538 ++++++++++++++++++++- src/server/issue_request.c | 39 +- src/server/issue_request.h | 2 - src/server/job_func.c | 12 +- src/server/node_manager.c | 32 +- src/server/node_power_state.c | 2 +- src/server/process_alps_status.c | 16 +- src/server/process_request.c | 386 +++------------ src/server/reply_send.c | 10 +- src/server/req_delete.c | 252 +--------- src/server/req_deletearray.c | 3 +- src/server/req_holdarray.c | 1 - src/server/req_holdjob.c | 32 +- src/server/req_jobobit.c | 31 +- src/server/req_manager.c | 4 +- src/server/req_message.c | 16 +- src/server/req_modify.c | 113 ++--- src/server/req_quejob.c | 8 +- src/server/req_register.c | 60 +-- src/server/req_rerun.c | 17 +- src/server/req_runjob.c | 147 ++---- src/server/req_shutdown.c | 2 - src/server/req_signal.c | 103 ++-- src/server/req_stat.c | 46 +- src/server/req_track.c | 22 +- src/server/svr_process_request.h | 4 - src/test/array_func/scaffolding.c | 7 + src/test/batch_request/scaffolding.c | 7 + src/test/batch_request/test_uut.c | 53 +- src/test/catch_child/scaffolding.c | 4 + src/test/display_alps_status/scaffolding.c | 3 + src/test/issue_request/scaffolding.c | 41 +- src/test/issue_request/test_uut.c | 57 ++- src/test/job_func/scaffolding.c | 12 +- src/test/job_func/test_uut.c | 2 +- src/test/job_recov/scaffolding.c | 22 +- src/test/mom_req_quejob/scaffolding.c | 3 + src/test/mom_req_quejob/test_uut.c | 1 - src/test/node_manager/scaffolding.c | 4 + src/test/node_manager/test_uut.c | 6 +- src/test/process_alps_status/scaffolding.c | 4 + src/test/process_request/scaffolding.c | 16 +- src/test/process_request/test_uut.c | 45 +- src/test/req_delete/scaffolding.c | 27 +- src/test/req_delete/test_uut.c | 61 +-- src/test/req_getcred/scaffolding.c | 9 + src/test/req_getcred/test_uut.c | 1 - src/test/req_holdjob/scaffolding.c | 13 + src/test/req_jobobit/scaffolding.c | 29 +- src/test/req_jobobit/test_uut.c | 5 +- src/test/req_manager/scaffolding.c | 4 + src/test/req_message/scaffolding.c | 12 +- src/test/req_modify/scaffolding.c | 19 + src/test/req_quejob/scaffolding.c | 7 + src/test/req_quejob/test_uut.c | 20 +- src/test/req_register/scaffolding.c | 19 +- src/test/req_register/test_uut.c | 16 +- src/test/req_rerun/scaffolding.c | 16 +- src/test/req_rerun/test_uut.c | 2 - src/test/req_runjob/scaffolding.c | 13 + src/test/req_runjob/test_uut.c | 2 - src/test/req_signal/scaffolding.c | 16 + src/test/req_signal/test_uut.c | 1 - src/test/req_stat/scaffolding.c | 18 + src/test/req_stat/test_uut.c | 1 - src/test/req_track/scaffolding.c | 14 + src/test/requests/scaffolding.c | 2 + 79 files changed, 1339 insertions(+), 1355 deletions(-) diff --git a/src/include/attribute.h b/src/include/attribute.h index 15aa7c97b6..734ea84c8b 100644 --- a/src/include/attribute.h +++ b/src/include/attribute.h @@ -147,7 +147,6 @@ typedef struct svrattrl svrattrl; #define al_op al_atopl.op - /* * The value of an attribute is contained in the following structure. * diff --git a/src/include/batch_request.h b/src/include/batch_request.h index e9aa75ba68..ede71794d3 100644 --- a/src/include/batch_request.h +++ b/src/include/batch_request.h @@ -319,8 +319,10 @@ struct rq_jobobit * so here is the union ... */ -struct batch_request + +class batch_request { + public: list_link rq_link; /* linkage of all requests */ int rq_type; /* type of request */ int rq_perm; /* access permissions for the user */ @@ -333,10 +335,10 @@ struct batch_request char rq_host[PBS_MAXHOSTNAME+1]; /* name of host sending request */ int rq_refcount; void *rq_extra; /* optional ptr to extra info */ - int rq_noreply; /* Set true if no reply is required */ + bool rq_noreply; /* Set true if no reply is required */ int rq_failcode; char *rq_extend; /* request "extension" data */ - char *rq_id; /* the batch request's id */ + std::string rq_id; /* the batch request's id */ struct batch_reply rq_reply; /* the reply area for this request */ @@ -394,9 +396,16 @@ struct batch_request struct rq_jobobit rq_jobobit; } rq_ind; - }; -typedef struct batch_request batch_request; + batch_request(); + batch_request(int type); + batch_request(const batch_request &other); + ~batch_request(); + batch_request &operator =(const batch_request &other); + + void update_object_id(int job_index); + int copy_attribute_list(const batch_request &other); + }; typedef container::item_container batch_request_holder; typedef container::item_container::item_iterator batch_request_holder_iterator; @@ -404,7 +413,6 @@ typedef container::item_container::item_iterator batch_reques extern batch_request_holder brh; -batch_request *alloc_br (int type); extern void reply_ack (struct batch_request *); extern void req_reject (int code, int aux, struct batch_request *, const char *, const char *); extern void reply_badattr (int code, int aux, svrattrl *, struct batch_request *); @@ -419,9 +427,9 @@ int process_request(struct tcp_chan *chan); int get_batch_request_id(batch_request *preq); int insert_batch_request(batch_request *preq); -batch_request *get_batch_request(char *br_id); -batch_request *get_remove_batch_request(char *br_id); -int remove_batch_request(char *br_id); +batch_request *get_batch_request(const char *br_id); +batch_request *get_remove_batch_request(const char *br_id); +int remove_batch_request(const char *br_id); int req_gpuctrl_svr(struct batch_request *preq); @@ -484,5 +492,4 @@ extern int encode_DIS_svrattrl (struct tcp_chan *chan, svrattrl *); extern int dis_request_read (struct tcp_chan *chan, struct batch_request *); extern int dis_reply_read (struct tcp_chan *chan, struct batch_reply *); -batch_request *duplicate_request(batch_request *preq, int job_index = -1); #endif /* BATCH_REQUEST_H */ diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 985ac92416..56f849fba5 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -1337,6 +1337,8 @@ bool have_reservation(job *, struct pbs_queue *); int lock_ji_mutex(job *pjob, const char *id, const char *msg, int logging); int unlock_ji_mutex(job *pjob, const char *id, const char *msg, int logging); +int issue_signal(job **, const char *, void(*)(struct batch_request *), void *, char *); + #ifdef BATCH_REQUEST_H extern job *chk_job_request(char *, struct batch_request *); extern int net_move(job *, struct batch_request *); @@ -1345,7 +1347,6 @@ extern int svr_chk_owner(struct batch_request *, job *); extern struct batch_request *cpy_stage(struct batch_request *, job *, enum job_atr, int); extern struct batch_request *setup_cpyfiles(struct batch_request *, job *, char *, char *, int, int); extern struct batch_request *cpy_checkpoint(struct batch_request *, job *, enum job_atr, int); -int issue_signal(job **, const char *, void(*)(struct batch_request *), void *, char *); #endif /* BATCH_REQUEST_H */ diff --git a/src/include/work_task.h b/src/include/work_task.h index e24e9fbb6f..9a44729a80 100644 --- a/src/include/work_task.h +++ b/src/include/work_task.h @@ -176,7 +176,7 @@ int insert_timed_task(all_tasks *, time_t task_time, work_task *); struct batch_request; -int issue_to_svr(const char *server_name, batch_request **preq, void (*f)(struct work_task *)); +int issue_to_svr(const char *server_name, batch_request *preq, void (*f)(struct work_task *)); #define TASKS_TO_REMOVE 1000 diff --git a/src/lib/Libattr/attr_func.c b/src/lib/Libattr/attr_func.c index eb69af002c..31af7c16b9 100644 --- a/src/lib/Libattr/attr_func.c +++ b/src/lib/Libattr/attr_func.c @@ -405,7 +405,6 @@ svrattrl *attrlist_create( - /* * free_attrlist - free the space allocated to a list of svrattrl * structures diff --git a/src/lib/Libifl/dec_svrattrl.c b/src/lib/Libifl/dec_svrattrl.c index f44e2ba3fd..369e92e69a 100644 --- a/src/lib/Libifl/dec_svrattrl.c +++ b/src/lib/Libifl/dec_svrattrl.c @@ -207,14 +207,15 @@ int decode_DIS_svrattrl( psvrat->al_op = (enum batch_op)disrui(chan, &rc); - if (rc) break; + if (rc) + break; append_link(phead, &psvrat->al_link, psvrat); } if (rc) { - (void)free(psvrat); + free(psvrat); } return (rc); diff --git a/src/lib/Libnet/net_common.c b/src/lib/Libnet/net_common.c index 7672ea5b14..2e1bfe7e6c 100644 --- a/src/lib/Libnet/net_common.c +++ b/src/lib/Libnet/net_common.c @@ -941,7 +941,7 @@ int pbs_getaddrinfo( if ((*ppAddrInfoOut = get_cached_addrinfo_full(pNode)) != NULL) { - return 0; + return(PBSE_NONE); } if (pHints == NULL) @@ -955,20 +955,20 @@ int pbs_getaddrinfo( { if (addrFound) { - rc = 0; + rc = PBSE_NONE; } else { rc = getaddrinfo(pNode,NULL,pHints,ppAddrInfoOut); } - if (rc == 0) + if (rc == PBSE_NONE) { addrFound = TRUE; *ppAddrInfoOut = insert_addr_name_info(*ppAddrInfoOut,pNode); if (*ppAddrInfoOut != NULL) { - return 0; + return(PBSE_NONE); } rc = EAI_AGAIN; } @@ -984,6 +984,7 @@ int pbs_getaddrinfo( } /* END pbs_getaddrinfo() */ + int connect_to_trqauthd( int *sock) diff --git a/src/resmom/catch_child.c b/src/resmom/catch_child.c index 8d080abe8c..3b13189454 100644 --- a/src/resmom/catch_child.c +++ b/src/resmom/catch_child.c @@ -1174,7 +1174,7 @@ int send_job_obit( /* send the job obiturary notice to the server */ - preq = alloc_br(PBS_BATCH_JobObit); + preq = new batch_request(PBS_BATCH_JobObit); if (preq == NULL) { @@ -1490,7 +1490,7 @@ void *obit_reply( /* read and decode the reply */ - if ((preq = alloc_br(PBS_BATCH_JobObit)) == NULL) + if ((preq = new batch_request(PBS_BATCH_JobObit)) == NULL) return(NULL); CLEAR_HEAD(preq->rq_ind.rq_jobobit.rq_attr); diff --git a/src/resmom/mom_process_request.c b/src/resmom/mom_process_request.c index 46c0909c1a..90c100d036 100644 --- a/src/resmom/mom_process_request.c +++ b/src/resmom/mom_process_request.c @@ -133,7 +133,7 @@ void *mom_process_request( time_now = time(NULL); - if ((request = alloc_br(0)) == NULL) + if ((request = new batch_request(0)) == NULL) { mom_close_client(sfds); return NULL; @@ -502,47 +502,25 @@ static void mom_close_client( +batch_request::batch_request( -/* - * alloc_br - allocate and clear a batch_request structure - */ - -struct batch_request *alloc_br( - - int type) + int type) : rq_type(type), rq_perm(0), rq_fromsvr(0), rq_conn(-1), rq_orgconn(-1), rq_extsz(0), + rq_time(time_now), rq_refcount(0), rq_extra(NULL), rq_noreply(FALSE), rq_failcode(0), + rq_extend(NULL), rq_id() { + CLEAR_LINK(this->rq_link); - struct batch_request *req = NULL; - - req = (struct batch_request *)calloc(1, sizeof(struct batch_request)); - - if (req == NULL) - { - log_err(errno, "alloc_br", msg_err_malloc); - - return(NULL); - } - - req->rq_type = type; - - req->rq_conn = -1; /* indicate not connected */ - req->rq_orgconn = -1; /* indicate not connected */ - req->rq_time = time_now; - req->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; - req->rq_noreply = FALSE; /* indicate reply is needed */ + memset(&this->rq_user, 0, sizeof(this->rq_user)); + memset(&this->rq_host, 0, sizeof(this->rq_host)); + memset(&this->rq_reply, 0, sizeof(this->rq_reply)); - CLEAR_LINK(req->rq_link); - - append_link(&svr_requests, &req->rq_link, req); - - return(req); + this->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; + append_link(&svr_requests, &this->rq_link, this); } - - /* * close_quejob - locate and deal with the new job that was being recevied * when the net connection closed. @@ -586,43 +564,31 @@ static void close_quejob( - - -/* - * free_br - free space allocated to a batch_request structure - * including any sub-structures - */ - -void free_br( - - batch_request *preq) +batch_request::~batch_request() { - if (preq == NULL) - return; - - delete_link(&preq->rq_link); + delete_link(&this->rq_link); - reply_free(&preq->rq_reply); + reply_free(&this->rq_reply); - if (preq->rq_extend) - free(preq->rq_extend); + if (this->rq_extend) + free(this->rq_extend); - if (preq->rq_extra) - free(preq->rq_extra); + if (this->rq_extra) + free(this->rq_extra); - switch (preq->rq_type) + switch (this->rq_type) { case PBS_BATCH_QueueJob: - free_attrlist(&preq->rq_ind.rq_queuejob.rq_attr); + free_attrlist(&this->rq_ind.rq_queuejob.rq_attr); break; case PBS_BATCH_JobCred: - if (preq->rq_ind.rq_jobcred.rq_data) - free(preq->rq_ind.rq_jobcred.rq_data); + if (this->rq_ind.rq_jobcred.rq_data) + free(this->rq_ind.rq_jobcred.rq_data); break; @@ -630,27 +596,27 @@ void free_br( case PBS_BATCH_jobscript: - if (preq->rq_ind.rq_jobfile.rq_data) - free(preq->rq_ind.rq_jobfile.rq_data); + if (this->rq_ind.rq_jobfile.rq_data) + free(this->rq_ind.rq_jobfile.rq_data); break; case PBS_BATCH_HoldJob: - freebr_manage(&preq->rq_ind.rq_hold.rq_orig); + freebr_manage(&this->rq_ind.rq_hold.rq_orig); break; case PBS_BATCH_CheckpointJob: - freebr_manage(&preq->rq_ind.rq_manager); + freebr_manage(&this->rq_ind.rq_manager); break; case PBS_BATCH_MessJob: - if (preq->rq_ind.rq_message.rq_text) - free(preq->rq_ind.rq_message.rq_text); + if (this->rq_ind.rq_message.rq_text) + free(this->rq_ind.rq_message.rq_text); break; @@ -658,7 +624,7 @@ void free_br( case PBS_BATCH_AsyModifyJob: - freebr_manage(&preq->rq_ind.rq_modify); + freebr_manage(&this->rq_ind.rq_modify); break; @@ -671,13 +637,13 @@ void free_br( case PBS_BATCH_StatusSvr: /* DIAGTODO: handle PBS_BATCH_StatusDiag */ - free_attrlist(&preq->rq_ind.rq_status.rq_attr); + free_attrlist(&this->rq_ind.rq_status.rq_attr); break; case PBS_BATCH_JobObit: - free_attrlist(&preq->rq_ind.rq_jobobit.rq_attr); + free_attrlist(&this->rq_ind.rq_jobobit.rq_attr); break; @@ -685,7 +651,7 @@ void free_br( case PBS_BATCH_DelFiles: - freebr_cpyfile(&preq->rq_ind.rq_cpyfile); + freebr_cpyfile(&this->rq_ind.rq_cpyfile); break; @@ -694,11 +660,25 @@ void free_br( /* NO-OP */ break; - } /* END switch (preq->rq_type) */ + } /* END switch (this->rq_type) */ + } // END destructor() - free(preq); - return; + +/* + * free_br - free space allocated to a batch_request structure + * including any sub-structures + */ + +void free_br( + + batch_request *preq) + + { + if (preq == NULL) + return; + + delete preq; } /* END free_br() */ diff --git a/src/resmom/mom_process_request.h b/src/resmom/mom_process_request.h index 123d5ff319..60ce6c73d1 100644 --- a/src/resmom/mom_process_request.h +++ b/src/resmom/mom_process_request.h @@ -10,8 +10,6 @@ void mom_dispatch_request(int sfds, struct batch_request *request); /* static void mom_close_client(int sfds); */ -struct batch_request *alloc_br(int type); - /* static void close_quejob(int sfds); */ void free_br(struct batch_request *preq); diff --git a/src/resmom/requests.c b/src/resmom/requests.c index 01aaea402c..a5a034d648 100644 --- a/src/resmom/requests.c +++ b/src/resmom/requests.c @@ -622,7 +622,7 @@ int return_file( return(0); } - prq = alloc_br(PBS_BATCH_MvJobFile); + prq = new batch_request(PBS_BATCH_MvJobFile); if (prq == NULL) { @@ -4333,7 +4333,7 @@ batch_request *initialize_stageout_request( job *pjob) { - batch_request *preq = alloc_br(PBS_BATCH_CopyFiles); + batch_request *preq = new batch_request(PBS_BATCH_CopyFiles); struct rq_cpyfile *pcf = &preq->rq_ind.rq_cpyfile; pcf->rq_dir = STAGE_DIR_OUT; diff --git a/src/server/array_func.c b/src/server/array_func.c index 5ddd36b99c..f794048f1c 100644 --- a/src/server/array_func.c +++ b/src/server/array_func.c @@ -63,7 +63,6 @@ extern int array_upgrade(job_array *, int, int, int *); extern char *get_correct_jobname(const char *jobid); extern int count_user_queued_jobs(pbs_queue *,char *); -extern void post_modify_arrayreq(batch_request *preq); /* global data items used */ @@ -1801,10 +1800,11 @@ int modify_array_range( } else { - struct batch_request *array_req = duplicate_request(preq, index); + batch_request *array_req = new batch_request(*preq); + array_req->update_object_id(index); mutex_mgr pjob_mutex = mutex_mgr(pjob->ji_mutex, true); pthread_mutex_unlock(pa->ai_mutex); - array_req->rq_noreply = TRUE; + array_req->rq_noreply = true; modify_job((void **)&pjob, plist, array_req, checkpoint_req, NO_MOM_RELAY); pa = get_jobs_array(&pjob); diff --git a/src/server/batch_request.c b/src/server/batch_request.c index 0800a3eb30..c7e95123d9 100644 --- a/src/server/batch_request.c +++ b/src/server/batch_request.c @@ -108,8 +108,7 @@ int get_batch_request_id( snprintf(buf, sizeof(buf), "%d", id); - if ((preq->rq_id = strdup(buf)) == NULL) - return(ENOMEM); + preq->rq_id = buf; insert_batch_request(preq); @@ -126,7 +125,7 @@ int insert_batch_request( brh.lock(); - if(!brh.insert(preq,preq->rq_id)) + if (!brh.insert(preq, preq->rq_id)) { rc = ENOMEM; log_err(rc, __func__, "No memory to resize the array...SYSTEM FAILURE\n"); @@ -150,7 +149,7 @@ int insert_batch_request( batch_request *get_batch_request( - char *br_id) + const char *br_id) { batch_request *preq = NULL; @@ -173,7 +172,7 @@ batch_request *get_batch_request( batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { batch_request *preq = NULL; @@ -194,7 +193,7 @@ batch_request *get_remove_batch_request( int remove_batch_request( - char *br_id) + const char *br_id) { brh.lock(); @@ -209,4 +208,531 @@ int remove_batch_request( +// Constructor with a type +batch_request::batch_request( + int type) : rq_type(type), rq_perm(0), rq_fromsvr(0), rq_conn(-1), rq_orgconn(-1), rq_extsz(0), + rq_time(time(NULL)), rq_refcount(0), rq_extra(NULL), rq_noreply(false), + rq_failcode(0), rq_extend(NULL), rq_id() + + { + memset(this->rq_user, 0, sizeof(this->rq_user)); + memset(this->rq_host, 0, sizeof(this->rq_host)); + memset(&this->rq_ind, 0, sizeof(this->rq_ind)); + memset(&this->rq_reply, 0, sizeof(this->rq_reply)); + + this->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; + } + + + +// Empty constructor +batch_request::batch_request() : rq_type(-1), rq_perm(0), rq_fromsvr(0), rq_conn(-1), + rq_orgconn(-1), rq_extsz(0), rq_time(time(NULL)), rq_refcount(0), + rq_extra(NULL), rq_noreply(false), rq_failcode(0), + rq_extend(NULL), rq_id() + + { + memset(this->rq_user, 0, sizeof(this->rq_user)); + memset(this->rq_host, 0, sizeof(this->rq_host)); + memset(&this->rq_ind, 0, sizeof(this->rq_ind)); + memset(&this->rq_reply, 0, sizeof(this->rq_reply)); + + this->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; + } + + + +int batch_request::copy_attribute_list( + + const batch_request &other) + + { + svrattrl *pal = (svrattrl *)GET_NEXT(other.rq_ind.rq_manager.rq_attr); + tlist_head *phead = &this->rq_ind.rq_manager.rq_attr; + svrattrl *newpal = NULL; + + while (pal != NULL) + { + newpal = (svrattrl *)calloc(1, pal->al_tsize + 1); + if (!newpal) + { + return(PBSE_SYSTEM); + } + + CLEAR_LINK(newpal->al_link); + + newpal->al_atopl.next = 0; + newpal->al_tsize = pal->al_tsize + 1; + newpal->al_nameln = pal->al_nameln; + newpal->al_flags = pal->al_flags; + newpal->al_atopl.name = (char *)newpal + sizeof(svrattrl); + strcpy((char *)newpal->al_atopl.name, pal->al_atopl.name); + newpal->al_nameln = pal->al_nameln; + newpal->al_atopl.resource = newpal->al_atopl.name + newpal->al_nameln; + + if (pal->al_atopl.resource != NULL) + strcpy((char *)newpal->al_atopl.resource, pal->al_atopl.resource); + + newpal->al_rescln = pal->al_rescln; + newpal->al_atopl.value = newpal->al_atopl.name + newpal->al_nameln + newpal->al_rescln; + strcpy((char *)newpal->al_atopl.value, pal->al_atopl.value); + newpal->al_valln = pal->al_valln; + newpal->al_atopl.op = pal->al_atopl.op; + + pal = (struct svrattrl *)GET_NEXT(pal->al_link); + } + + if ((phead != NULL) && + (newpal != NULL)) + append_link(phead, &newpal->al_link, newpal); + + return(PBSE_NONE); + } /* END copy_attribute_list() */ + + + +void batch_request::update_object_id( + + int job_index) + + { + if (job_index >= 0) + { + /* If this is a job array it is possible we only have the array name + and not the individual job. We need to find out what we have and + modify the name if needed */ + char newjobname[PBS_MAXSVRJOBID+1]; + char *ptr1 = strstr(this->rq_ind.rq_manager.rq_objname, "[]"); + + if (ptr1) + { + char *ptr2; + ptr1++; + strcpy(newjobname, this->rq_ind.rq_manager.rq_objname); + + if ((ptr2 = strstr(newjobname, "[]")) != NULL) + { + ptr2++; + *ptr2 = 0; + } + + sprintf(this->rq_ind.rq_manager.rq_objname,"%s%d%s", + newjobname, job_index, ptr1); + } + } + } // END update_object_id() + + + +// Copy constructor +batch_request::batch_request( + + const batch_request &other) : rq_type(other.rq_type), rq_perm(other.rq_perm), + rq_fromsvr(other.rq_fromsvr), rq_conn(other.rq_conn), + rq_orgconn(other.rq_orgconn), rq_extsz(other.rq_extsz), + rq_time(other.rq_time), rq_refcount(other.rq_refcount), + rq_noreply(other.rq_noreply), rq_failcode(other.rq_failcode), + rq_id(other.rq_id) + + { + if (other.rq_extra == NULL) + this->rq_extra = NULL; + else + this->rq_extra = strdup((char *)other.rq_extra); + + if (other.rq_extend == NULL) + this->rq_extend = NULL; + else + this->rq_extend = strdup(other.rq_extend); + + strcpy(this->rq_user, other.rq_user); + strcpy(this->rq_host, other.rq_host); + memcpy(&this->rq_reply, &other.rq_reply, sizeof(this->rq_reply)); + + switch (this->rq_type) + { + /* This function was created for a modify array request (PBS_BATCH_ModifyJob) + the other.rq_ind structure was allocated in dis_request_read. If other + BATCH types are needed refer to that function to see how the rq_ind structure + was allocated and then copy it here. */ + case PBS_BATCH_DeleteJob: + case PBS_BATCH_HoldJob: + case PBS_BATCH_CheckpointJob: + case PBS_BATCH_ModifyJob: + case PBS_BATCH_AsyModifyJob: + + memcpy(this->rq_ind.rq_manager.rq_objname, + other.rq_ind.rq_manager.rq_objname, PBS_MAXSVRJOBID + 1); + + /* based on how decode_DIS_Manage allocates data */ + CLEAR_HEAD(this->rq_ind.rq_manager.rq_attr); + + this->rq_ind.rq_manager.rq_cmd = other.rq_ind.rq_manager.rq_cmd; + this->rq_ind.rq_manager.rq_objtype = other.rq_ind.rq_manager.rq_objtype; + + strcpy(this->rq_ind.rq_manager.rq_objname, other.rq_ind.rq_manager.rq_objname); + this->copy_attribute_list(other); + + break; + + case PBS_BATCH_SignalJob: + + strcpy(this->rq_ind.rq_signal.rq_jid, other.rq_ind.rq_signal.rq_jid); + strcpy(this->rq_ind.rq_signal.rq_signame, other.rq_ind.rq_signal.rq_signame); + + break; + + case PBS_BATCH_MessJob: + + strcpy(this->rq_ind.rq_message.rq_jid, other.rq_ind.rq_message.rq_jid); + this->rq_ind.rq_message.rq_file = other.rq_ind.rq_message.rq_file; + + if (other.rq_ind.rq_message.rq_text != NULL) + this->rq_ind.rq_message.rq_text = strdup(other.rq_ind.rq_message.rq_text); + + break; + + case PBS_BATCH_RunJob: + case PBS_BATCH_AsyrunJob: + + strcpy(this->rq_ind.rq_run.rq_jid, other.rq_ind.rq_run.rq_jid); + + if (other.rq_ind.rq_run.rq_destin) + this->rq_ind.rq_run.rq_destin = strdup(other.rq_ind.rq_run.rq_destin); + + break; + + case PBS_BATCH_Rerun: + + strcpy(this->rq_ind.rq_rerun, other.rq_ind.rq_rerun); + break; + + default: + + break; + } + } + + + +batch_request &batch_request::operator =( + + const batch_request &other) + + { + this->rq_type = other.rq_type; + this->rq_perm = other.rq_perm; + this->rq_fromsvr = other.rq_fromsvr; + this->rq_conn = other.rq_conn; + this->rq_orgconn = other.rq_orgconn; + this->rq_extsz = other.rq_extsz; + this->rq_time = other.rq_time; + this->rq_refcount = other.rq_refcount; + this->rq_noreply = other.rq_noreply; + this->rq_failcode = other.rq_failcode; + + if (other.rq_extra == NULL) + this->rq_extra = NULL; + else + this->rq_extra = strdup((char *)other.rq_extra); + + if (other.rq_extend == NULL) + this->rq_extend = NULL; + else + this->rq_extend = strdup(other.rq_extend); + + this->rq_id = other.rq_id; + + strcpy(this->rq_user, other.rq_user); + strcpy(this->rq_host, other.rq_host); + memcpy(&this->rq_reply, &other.rq_reply, sizeof(this->rq_reply)); + + switch (this->rq_type) + { + /* This function was created for a modify array request (PBS_BATCH_ModifyJob) + the other.rq_ind structure was allocated in dis_request_read. If other + BATCH types are needed refer to that function to see how the rq_ind structure + was allocated and then copy it here. */ + case PBS_BATCH_DeleteJob: + case PBS_BATCH_HoldJob: + case PBS_BATCH_CheckpointJob: + case PBS_BATCH_ModifyJob: + case PBS_BATCH_AsyModifyJob: + + memcpy(this->rq_ind.rq_manager.rq_objname, + other.rq_ind.rq_manager.rq_objname, PBS_MAXSVRJOBID + 1); + + /* based on how decode_DIS_Manage allocates data */ + CLEAR_HEAD(this->rq_ind.rq_manager.rq_attr); + + this->rq_ind.rq_manager.rq_cmd = other.rq_ind.rq_manager.rq_cmd; + this->rq_ind.rq_manager.rq_objtype = other.rq_ind.rq_manager.rq_objtype; + + strcpy(this->rq_ind.rq_manager.rq_objname, other.rq_ind.rq_manager.rq_objname); + this->copy_attribute_list(other); + + break; + + case PBS_BATCH_SignalJob: + + strcpy(this->rq_ind.rq_signal.rq_jid, other.rq_ind.rq_signal.rq_jid); + strcpy(this->rq_ind.rq_signal.rq_signame, other.rq_ind.rq_signal.rq_signame); + + break; + + case PBS_BATCH_MessJob: + + strcpy(this->rq_ind.rq_message.rq_jid, other.rq_ind.rq_message.rq_jid); + this->rq_ind.rq_message.rq_file = other.rq_ind.rq_message.rq_file; + + if (other.rq_ind.rq_message.rq_text != NULL) + this->rq_ind.rq_message.rq_text = strdup(other.rq_ind.rq_message.rq_text); + + break; + + case PBS_BATCH_RunJob: + case PBS_BATCH_AsyrunJob: + + strcpy(this->rq_ind.rq_run.rq_jid, other.rq_ind.rq_run.rq_jid); + if (other.rq_ind.rq_run.rq_destin) + this->rq_ind.rq_run.rq_destin = strdup(other.rq_ind.rq_run.rq_destin); + + break; + + case PBS_BATCH_Rerun: + + strcpy(this->rq_ind.rq_rerun, other.rq_ind.rq_rerun); + break; + + default: + + break; + } + + return(*this); + } // END = operator + + + +void freebr_manage( + + struct rq_manage *pmgr) + + { + free_attrlist(&pmgr->rq_attr); + + return; + } /* END freebr_manage() */ + + + +void freebr_cpyfile( + + struct rq_cpyfile *pcf) + + { + struct rqfpair *ppair; + + while ((ppair = (struct rqfpair *)GET_NEXT(pcf->rq_pair)) != NULL) + { + delete_link(&ppair->fp_link); + + if (ppair->fp_local != NULL) + free(ppair->fp_local); + + if (ppair->fp_rmt != NULL) + free(ppair->fp_rmt); + + free(ppair); + } + + return; + } /* END freebr_cpyfile() */ + + + +void free_rescrq( + + struct rq_rescq *pq) + + { + int i; + + i = pq->rq_num; + + while (i--) + { + if (*(pq->rq_list + i) != NULL) + free(*(pq->rq_list + i)); + } + + if (pq->rq_list != NULL) + free(pq->rq_list); + + return; + } /* END free_rescrq() */ + + + +batch_request::~batch_request() + + { + if (this->rq_id.size() != 0) + { + remove_batch_request(this->rq_id.c_str()); + } + + reply_free(&this->rq_reply); + + if (this->rq_extend) + { + free(this->rq_extend); + this->rq_extend = NULL; + } + + if (this->rq_extra) + { + free(this->rq_extra); + this->rq_extra = NULL; + } + + switch (this->rq_type) + { + case PBS_BATCH_QueueJob: + case PBS_BATCH_QueueJob2: + + free_attrlist(&this->rq_ind.rq_queuejob.rq_attr); + + break; + + case PBS_BATCH_JobCred: + + if (this->rq_ind.rq_jobcred.rq_data) + { + free(this->rq_ind.rq_jobcred.rq_data); + this->rq_ind.rq_jobcred.rq_data = NULL; + } + + break; + + case PBS_BATCH_MvJobFile: + case PBS_BATCH_jobscript: + case PBS_BATCH_jobscript2: + + if (this->rq_ind.rq_jobfile.rq_data) + { + free(this->rq_ind.rq_jobfile.rq_data); + this->rq_ind.rq_jobfile.rq_data = NULL; + } + break; + + case PBS_BATCH_HoldJob: + + freebr_manage(&this->rq_ind.rq_hold.rq_orig); + + break; + + case PBS_BATCH_CheckpointJob: + + freebr_manage(&this->rq_ind.rq_manager); + + break; + + case PBS_BATCH_MessJob: + + if (this->rq_ind.rq_message.rq_text) + { + free(this->rq_ind.rq_message.rq_text); + this->rq_ind.rq_message.rq_text = NULL; + } + + break; + + case PBS_BATCH_ModifyJob: + + case PBS_BATCH_AsyModifyJob: + + freebr_manage(&this->rq_ind.rq_modify); + + break; + + case PBS_BATCH_StatusJob: + + case PBS_BATCH_StatusQue: + + case PBS_BATCH_StatusNode: + + case PBS_BATCH_StatusSvr: + /* DIAGTODO: handle PBS_BATCH_StatusDiag */ + + free_attrlist(&this->rq_ind.rq_status.rq_attr); + + break; + + case PBS_BATCH_JobObit: + + free_attrlist(&this->rq_ind.rq_jobobit.rq_attr); + + break; + + case PBS_BATCH_CopyFiles: + + case PBS_BATCH_DelFiles: + + freebr_cpyfile(&this->rq_ind.rq_cpyfile); + + break; + + case PBS_BATCH_ModifyNode: + + case PBS_BATCH_Manager: + + freebr_manage(&this->rq_ind.rq_manager); + + break; + + case PBS_BATCH_ReleaseJob: + + freebr_manage(&this->rq_ind.rq_release); + + break; + + case PBS_BATCH_Rescq: + + free_rescrq(&this->rq_ind.rq_rescq); + + break; + + case PBS_BATCH_SelectJobs: + + case PBS_BATCH_SelStat: + + free_attrlist(&this->rq_ind.rq_select); + + break; + + case PBS_BATCH_SelStatAttr: + + free_attrlist(&this->rq_ind.rq_select); + free_attrlist(&this->rq_ind.rq_status.rq_attr); + + break; + + case PBS_BATCH_RunJob: + case PBS_BATCH_AsyrunJob: + + if (this->rq_ind.rq_run.rq_destin) + { + free(this->rq_ind.rq_run.rq_destin); + this->rq_ind.rq_run.rq_destin = NULL; + } + break; + + default: + + /* NO-OP */ + + break; + } /* END switch (this->rq_type) */ + } diff --git a/src/server/issue_request.c b/src/server/issue_request.c index 788b62b2b8..87215a0968 100644 --- a/src/server/issue_request.c +++ b/src/server/issue_request.c @@ -145,9 +145,9 @@ int issue_Drequest(int conn, struct batch_request *request); int relay_to_mom( - job **pjob_ptr, - struct batch_request *request, /* the request to send */ - void (*func)(struct work_task *)) + job **pjob_ptr, + batch_request *request, /* the request to send */ + void (*func)(struct work_task *)) { int handle; /* a client style connection handle */ @@ -261,6 +261,8 @@ void reissue_to_svr( br_id = (char *)pwt->wt_parm1; preq = get_remove_batch_request(br_id); + free(br_id); + /* if not timed-out, retry send to remote server */ if (preq != NULL) { @@ -274,7 +276,7 @@ void reissue_to_svr( } if (((time_now - preq->rq_time) > PBS_NET_RETRY_LIMIT) || - (issue_to_svr(serverName, &preq, pwt->wt_parmfunc) != PBSE_NONE)) + (issue_to_svr(serverName, preq, pwt->wt_parmfunc) != PBSE_NONE)) { /* either timed-out or got hard error, tell post-function */ @@ -284,6 +286,8 @@ void reissue_to_svr( if (pwt->wt_parmfunc != NULL) (* pwt->wt_parmfunc)(pwt); } + + delete preq; } if (serverName) @@ -302,12 +306,12 @@ void queue_a_retry_task( { /* create a new batch_request because preq is going to be freed when issue_to_svr returns success */ - batch_request *new_preq = duplicate_request(preq, -1); + batch_request *new_preq = new batch_request(*preq); struct work_task *pwt; get_batch_request_id(new_preq); - pwt = set_task(WORK_Timed, (time(NULL) + PBS_NET_RETRY_TIME), reissue_to_svr, new_preq->rq_id, TRUE); + pwt = set_task(WORK_Timed, (time(NULL) + PBS_NET_RETRY_TIME), reissue_to_svr, strdup(new_preq->rq_id.c_str()), TRUE); pwt->wt_parmfunc = replyfunc; @@ -326,13 +330,17 @@ void queue_a_retry_task( * -1 on permanent error (no such host) * * On temporary error, establish a work_task to retry after a delay. + * + * @param servern - the name of the destination server for this request + * @param preq - the batch request we're sending + * @param replyfunc - function that should be called after sending the request */ int issue_to_svr( - const char *servern, /* I */ - struct batch_request **preq_ptr, /* I */ - void (*replyfunc) (struct work_task *)) /* I */ + const char *servern, + batch_request *preq, + void (*replyfunc)(struct work_task *)) { int rc = PBSE_NONE; @@ -342,7 +350,6 @@ int issue_to_svr( pbs_net_t svraddr; char *svrname; unsigned int port = pbs_server_port_dis; - batch_request *preq = *preq_ptr; snprintf(preq->rq_host, sizeof(preq->rq_host), "%s", servern); @@ -372,12 +379,8 @@ int issue_to_svr( if (((rc = issue_Drequest(handle, preq, true)) == PBSE_NONE) && (handle != PBS_LOCAL_CONNECTION)) { - /* preq is already freed if handle == PBS_LOCAL_CONNECTION - a reply - * has always been sent */ rc = preq->rq_reply.brp_code; } - else if (handle == PBS_LOCAL_CONNECTION) - *preq_ptr = NULL; return(rc); } @@ -418,10 +421,10 @@ int que_to_local_svr(struct batch_request *preq) /* I */ preq->rq_fromsvr = 1; preq->rq_perm = ATR_DFLAG_MGRD | ATR_DFLAG_MGWR | ATR_DFLAG_SvWR; - if (preq->rq_id == NULL) + if (preq->rq_id.size() == 0) get_batch_request_id(preq); - set_task(WORK_Immed, 0, reissue_to_svr, preq->rq_id, TRUE); + set_task(WORK_Immed, 0, reissue_to_svr, strdup(preq->rq_id.c_str()), TRUE); return(PBSE_NONE); } /* END que_to_local_svr() */ @@ -439,6 +442,8 @@ int que_to_local_svr(struct batch_request *preq) /* I */ * * THIS SHOULD NOT BE USED IF AN EXTERNAL (CLIENT) REQUEST WAS "relayed". * The request/reply structure is still needed to reply to the client. + * + * FIXME: This also appears to be dead code */ void release_req( @@ -450,7 +455,7 @@ void release_req( char *br_id = (char *)pwt->wt_parm1; if ((preq = get_remove_batch_request(br_id)) != NULL) - free_br(preq); + delete preq; if (pwt->wt_event != -1) svr_disconnect(pwt->wt_event); diff --git a/src/server/issue_request.h b/src/server/issue_request.h index c9c7cb51fc..e2a3f332c4 100644 --- a/src/server/issue_request.h +++ b/src/server/issue_request.h @@ -12,6 +12,4 @@ void reissue_to_svr(struct work_task *); int handle_local_request(int conn, batch_request *request); -void release_req(struct work_task *pwt); - #endif /* _ISSUE_REQUEST_H */ diff --git a/src/server/job_func.c b/src/server/job_func.c index e0ce613c9c..b448fa23ec 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -142,7 +142,6 @@ #include "array.h" #include "pbs_job.h" #include "svr_func.h" /* get_svr_attr_* */ -#include "issue_request.h" /* release_req */ #include "ji_mutex.h" #include "user_info.h" #include "mutex_mgr.hpp" @@ -170,7 +169,6 @@ extern struct batch_request *setup_cpyfiles(struct batch_request *,job *,char*,c extern int job_log_open(char *, char *); extern int log_job_record(const char *buf); extern void check_job_log(struct work_task *ptask); -int issue_signal(job **, const char *, void(*)(batch_request *), void *, char *); void handle_complete_second_time(struct work_task *ptask); /* Local Private Functions */ @@ -547,7 +545,7 @@ int job_abt( { svr_setjobstate(pjob, JOB_STATE_RUNNING, JOB_SUBSTATE_ABORT, FALSE); - if ((rc = issue_signal(&pjob, "SIGKILL", free_br, NULL, NULL)) != 0) + if ((rc = issue_signal(&pjob, "SIGKILL", NULL, NULL, NULL)) != 0) { if (pjob != NULL) { @@ -1632,9 +1630,9 @@ void remove_checkpoint( job **pjob_ptr) /* I */ { - struct batch_request *preq = NULL; - char log_buf[LOCAL_LOG_BUF_SIZE]; - job *pjob = NULL; + batch_request *preq = NULL; + char log_buf[LOCAL_LOG_BUF_SIZE]; + job *pjob = NULL; if (pjob_ptr == NULL) { @@ -1684,7 +1682,7 @@ void remove_checkpoint( (char *)"unable to remove checkpoint file for job"); } - free_br(preq); + delete preq; } return; diff --git a/src/server/node_manager.c b/src/server/node_manager.c index d79512141f..0884bc626a 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -572,7 +572,6 @@ int kill_job_on_mom( struct pbsnode *pnode) { - batch_request *preq; int rc = -1; int conn; int local_errno = 0; @@ -587,27 +586,20 @@ int kill_job_on_mom( if (conn >= 0) { - if ((preq = alloc_br(PBS_BATCH_SignalJob)) == NULL) - { - log_err(-1, __func__, "unable to allocate SignalJob request-trouble!"); - svr_disconnect(conn); - } - else - { - snprintf(preq->rq_ind.rq_signal.rq_jid, sizeof(preq->rq_ind.rq_signal.rq_jid), "%s", job_id); - snprintf(preq->rq_ind.rq_signal.rq_signame, sizeof(preq->rq_ind.rq_signal.rq_signame), "SIGKILL"); - preq->rq_extra = strdup(SYNC_KILL); - pnode->tmp_unlock_node(__func__, NULL, LOGLEVEL); - rc = issue_Drequest(conn, preq, true); + batch_request preq(PBS_BATCH_SignalJob); + + snprintf(preq.rq_ind.rq_signal.rq_jid, sizeof(preq.rq_ind.rq_signal.rq_jid), "%s", job_id); + snprintf(preq.rq_ind.rq_signal.rq_signame, sizeof(preq.rq_ind.rq_signal.rq_signame), "SIGKILL"); + preq.rq_extra = strdup(SYNC_KILL); + pnode->tmp_unlock_node(__func__, NULL, LOGLEVEL); + rc = issue_Drequest(conn, &preq, true); - if (preq->rq_reply.brp_code == PBSE_TIMEOUT) - update_failure_counts(node_name.c_str(), PBSE_TIMEOUT); - else - update_failure_counts(node_name.c_str(), 0); + if (preq.rq_reply.brp_code == PBSE_TIMEOUT) + update_failure_counts(node_name.c_str(), PBSE_TIMEOUT); + else + update_failure_counts(node_name.c_str(), 0); - free_br(preq); - pnode->tmp_lock_node(__func__, NULL, LOGLEVEL); - } + pnode->tmp_lock_node(__func__, NULL, LOGLEVEL); } else { diff --git a/src/server/node_power_state.c b/src/server/node_power_state.c index b58a6a4600..03f6b7a769 100644 --- a/src/server/node_power_state.c +++ b/src/server/node_power_state.c @@ -243,7 +243,7 @@ int set_node_power_state( //Can't change the power state on a node with running jobs. return PBSE_CANT_CHANGE_POWER_STATE_WITH_JOBS_RUNNING; } - struct batch_request *request = alloc_br(PBS_BATCH_ChangePowerState); + batch_request *request = new batch_request(PBS_BATCH_ChangePowerState); if (request == NULL) { return PBSE_SYSTEM; diff --git a/src/server/process_alps_status.c b/src/server/process_alps_status.c index cc3be2a531..316878370f 100644 --- a/src/server/process_alps_status.c +++ b/src/server/process_alps_status.c @@ -215,7 +215,6 @@ void *check_if_orphaned( char *node_name = (char *)vp; char *rsv_id = NULL; std::string job_id; - batch_request *preq; int handle = -1; int retries = 0; struct pbsnode *pnode; @@ -250,20 +249,13 @@ void *check_if_orphaned( pnode->unlock_node(__func__, NULL, LOGLEVEL); } - if ((preq = alloc_br(PBS_BATCH_DeleteReservation)) == NULL) - { - free(node_name); - alps_reservations.remove_from_orphaned_list(rsv_id); - return(NULL); - } - - preq->rq_extend = strdup(rsv_id); - if ((pnode = get_next_login_node(NULL)) != NULL) { struct in_addr hostaddr; int local_errno; pbs_net_t momaddr; + batch_request preq(PBS_BATCH_DeleteReservation); + preq.rq_extend = strdup(rsv_id); memcpy(&hostaddr, &pnode->nd_sock_addr.sin_addr, sizeof(hostaddr)); momaddr = ntohl(hostaddr.s_addr); @@ -286,9 +278,7 @@ void *check_if_orphaned( pnode->unlock_node(__func__, NULL, LOGLEVEL); if (handle >= 0) - issue_Drequest(handle, preq, true); - - free_br(preq); + issue_Drequest(handle, &preq, true); } alps_reservations.remove_from_orphaned_list(rsv_id); diff --git a/src/server/process_request.c b/src/server/process_request.c index 4d4f43b578..519604c2f9 100644 --- a/src/server/process_request.c +++ b/src/server/process_request.c @@ -188,9 +188,6 @@ static const int munge_on = 1; static const int munge_on = 0; #endif -static void freebr_manage(struct rq_manage *); -static void freebr_cpyfile(struct rq_cpyfile *); -static void free_rescrq(struct rq_rescq *); void close_quejob(int sfds); /* END private prototypes */ @@ -332,13 +329,22 @@ bool request_passes_acl_check( -batch_request *read_request_from_socket( +/* + * read_request_from_socket() + * + * Initializes request from the data on the socket + * @param chan - the wrapper for the socket we're reading data from + * @param request - the request we're initializing + * @return PBSE_* to indicate what happened + */ - tcp_chan *chan) +int read_request_from_socket( + + tcp_chan *chan, + batch_request &request) { int rc = PBSE_NONE; - struct batch_request *request = NULL; char log_buf[LOCAL_LOG_BUF_SIZE]; time_t time_now = time(NULL); @@ -353,7 +359,7 @@ batch_request *read_request_from_socket( if ((sfds < 0) || (sfds >= PBS_NET_MAX_CONNECTIONS)) - return(NULL); + return(-1); pthread_mutex_lock(svr_conn[sfds].cn_mutex); conn_active = svr_conn[sfds].cn_active; @@ -365,10 +371,7 @@ batch_request *read_request_from_socket( svr_conn[sfds].cn_lasttime = time_now; pthread_mutex_unlock(svr_conn[sfds].cn_mutex); - if ((request = alloc_br(0)) == NULL) - return(NULL); - - request->rq_conn = sfds; + request.rq_conn = sfds; /* * Read in the request and decode it to the internal request structure. @@ -386,14 +389,14 @@ batch_request *read_request_from_socket( } #endif /* END ENABLE_UNIX_SOCKETS */ - rc = dis_request_read(chan, request); + rc = dis_request_read(chan, &request); if ((rc == PBSE_SYSTEM) || (rc == PBSE_INTERNAL) || (rc == PBSE_SOCKET_CLOSE)) { /* read error, likely cannot send reply so just disconnect, indicate permanent * failure by setting the type to PBS_BATCH_Disconnect */ - request->rq_type = PBS_BATCH_Disconnect; - return(request); + request.rq_type = PBS_BATCH_Disconnect; + return(rc); } else if (rc > 0) { @@ -403,8 +406,8 @@ batch_request *read_request_from_socket( * request didn't decode, either garbage or unknown * request type, in either case, return reject-reply */ - request->rq_failcode = rc; - return(request); + request.rq_failcode = rc; + return(rc); } } else @@ -419,11 +422,11 @@ batch_request *read_request_from_socket( "request on invalid type of connection (%d) from %s", conn_active, netaddr_long(conn_addr, out)); - req_reject(PBSE_BADHOST, 0, request, NULL, tmpLine); - return(NULL); + req_reject(PBSE_BADHOST, 0, &request, NULL, tmpLine); + return(-1); } - if (get_connecthost(sfds, request->rq_host, PBS_MAXHOSTNAME) != 0) + if (get_connecthost(sfds, request.rq_host, PBS_MAXHOSTNAME) != 0) { char ipstr[80]; sprintf(log_buf, "%s: %s", @@ -437,17 +440,17 @@ batch_request *read_request_from_socket( "cannot determine hostname for connection from %s", log_buf); - req_reject(PBSE_BADHOST, 0, request, NULL, tmpLine); - return(NULL); + req_reject(PBSE_BADHOST, 0, &request, NULL, tmpLine); + return(-1); } if (LOGLEVEL >= 8) { sprintf(log_buf, msg_request, - reqtype_to_txt(request->rq_type), - request->rq_user, - request->rq_host, + reqtype_to_txt(request.rq_type), + request.rq_user, + request.rq_host, sfds); log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_REQUEST, "", log_buf); @@ -455,22 +458,22 @@ batch_request *read_request_from_socket( /* is the request from a host acceptable to the server */ if (conn_socktype & PBS_SOCK_UNIX) - strcpy(request->rq_host, server_name); + strcpy(request.rq_host, server_name); - if (request_passes_acl_check(request, conn_addr) == false) + if (request_passes_acl_check(&request, conn_addr) == false) { /* See if the request is in the limited acl list */ - if (limited_acls.is_authorized(request->rq_host, request->rq_user) == false) + if (limited_acls.is_authorized(request.rq_host, request.rq_user) == false) { char tmpLine[MAXLINE]; snprintf(tmpLine, sizeof(tmpLine), "request not authorized from host %s", - request->rq_host); - req_reject(PBSE_BADHOST, 0, request, NULL, tmpLine); - return(NULL); + request.rq_host); + req_reject(PBSE_BADHOST, 0, &request, NULL, tmpLine); + return(-1); } } - return(request); + return(PBSE_NONE); } /* END read_request_from_socket() */ @@ -494,7 +497,7 @@ int process_request( { int rc = PBSE_NONE; - struct batch_request *request = NULL; + batch_request request; long state = SV_STATE_DOWN; time_t time_now = time(NULL); @@ -513,18 +516,13 @@ int process_request( svr_conn[sfds].cn_lasttime = time_now; pthread_mutex_unlock(svr_conn[sfds].cn_mutex); - request = read_request_from_socket(chan); + rc = read_request_from_socket(chan, request); - if (request == NULL) - rc = -1; - else if (request->rq_type == PBS_BATCH_Disconnect) + if (request.rq_type == PBS_BATCH_Disconnect) rc = PBSE_SOCKET_CLOSE; - else - rc = request->rq_failcode; if (rc != PBSE_NONE) { - free_br(request); return(rc); } @@ -537,9 +535,9 @@ int process_request( { /* request came from another server */ - request->rq_fromsvr = 1; + request.rq_fromsvr = 1; - request->rq_perm = + request.rq_perm = ATR_DFLAG_USRD | ATR_DFLAG_USWR | ATR_DFLAG_OPRD | ATR_DFLAG_OPWR | ATR_DFLAG_MGRD | ATR_DFLAG_MGWR | @@ -550,7 +548,7 @@ int process_request( /* request not from another server */ conn_credent[sfds].timestamp = time_now; - request->rq_fromsvr = 0; + request.rq_fromsvr = 0; /* * Client must be authenticated by an Authenticate User Request, if not, @@ -568,9 +566,9 @@ int process_request( * creds. Authorization is still granted in svr_get_privilege below */ - if (request->rq_type == PBS_BATCH_Connect) + if (request.rq_type == PBS_BATCH_Connect) { - return(req_connect(request)); + return(req_connect(&request)); } if (conn_socktype & PBS_SOCK_UNIX) @@ -586,14 +584,14 @@ int process_request( else if (munge_on) { /* If munge_on is true we will validate the connection now */ - if (request->rq_type == PBS_BATCH_AltAuthenUser) + if (request.rq_type == PBS_BATCH_AltAuthenUser) { - rc = req_altauthenuser(request); + rc = req_altauthenuser(&request); return(rc); } else { - rc = authenticate_user(request, &conn_credent[sfds], &auth_err); + rc = authenticate_user(&request, &conn_credent[sfds], &auth_err); } } #endif @@ -601,11 +599,11 @@ int process_request( /* skip checking user if we did not get an authenticated credential */ rc = PBSE_BADCRED; else - rc = authenticate_user(request, &conn_credent[sfds], &auth_err); + rc = authenticate_user(&request, &conn_credent[sfds], &auth_err); if (rc != 0) { - req_reject(rc, 0, request, NULL, auth_err); + req_reject(rc, 0, &request, NULL, auth_err); if (auth_err != NULL) free(auth_err); @@ -618,9 +616,9 @@ int process_request( * for root on the jobs execution node. */ - if (((request->rq_type == PBS_BATCH_ModifyJob) || - (request->rq_type == PBS_BATCH_ReleaseJob)) && - (strcmp(request->rq_user, PBS_DEFAULT_ADMIN) == 0)) + if (((request.rq_type == PBS_BATCH_ModifyJob) || + (request.rq_type == PBS_BATCH_ReleaseJob)) && + (strcmp(request.rq_user, PBS_DEFAULT_ADMIN) == 0)) { job *pjob; char *dptr; @@ -629,13 +627,13 @@ int process_request( /* make short host name */ - strcpy(short_host, request->rq_host); + strcpy(short_host, request.rq_host); if ((dptr = strchr(short_host, '.')) != NULL) { *dptr = '\0'; } - if ((pjob = svr_find_job(request->rq_ind.rq_modify.rq_objname, FALSE)) != NULL) + if ((pjob = svr_find_job(request.rq_ind.rq_modify.rq_objname, FALSE)) != NULL) { mutex_mgr pjob_mutex = mutex_mgr(pjob->ji_mutex, true); if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) @@ -647,7 +645,7 @@ int process_request( (csv_find_string(pjob->ji_wattr[JOB_ATR_checkpoint].at_val.at_str, "enabled") != NULL)) && (strstr(pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str, short_host) != NULL)) { - request->rq_perm = svr_get_privilege(request->rq_user, server_host); + request.rq_perm = svr_get_privilege(request.rq_user, server_host); skip = TRUE; } } @@ -656,18 +654,18 @@ int process_request( if (!skip) { - request->rq_perm = svr_get_privilege(request->rq_user, request->rq_host); + request.rq_perm = svr_get_privilege(request.rq_user, request.rq_host); } } else { - request->rq_perm = svr_get_privilege(request->rq_user, request->rq_host); + request.rq_perm = svr_get_privilege(request.rq_user, request.rq_host); } } /* END else (conn_authen == PBS_NET_CONN_FROM_PRIVIL) */ - if (threadpool_is_too_busy(request_pool, request->rq_perm)) + if (threadpool_is_too_busy(request_pool, request.rq_perm)) { - req_reject(PBSE_SERVER_BUSY, 0, request, NULL, NULL); + req_reject(PBSE_SERVER_BUSY, 0, &request, NULL, NULL); return(PBSE_SERVER_BUSY); } @@ -676,7 +674,7 @@ int process_request( if (state > SV_STATE_RUN) { - switch (request->rq_type) + switch (request.rq_type) { case PBS_BATCH_AsyrunJob: case PBS_BATCH_JobCred: @@ -688,7 +686,7 @@ int process_request( case PBS_BATCH_jobscript: case PBS_BATCH_jobscript2: - req_reject(PBSE_SVRDOWN, 0, request, NULL, NULL); + req_reject(PBSE_SVRDOWN, 0, &request, NULL, NULL); return(PBSE_SVRDOWN); } @@ -700,7 +698,7 @@ int process_request( * the request struture. */ - rc = dispatch_request(sfds, request); + rc = dispatch_request(sfds, &request); return(rc); } /* END process_request() */ @@ -1029,40 +1027,6 @@ int dispatch_request( - -/* - * alloc_br - allocate and clear a batch_request structure - */ - -struct batch_request *alloc_br( - - int type) - - { - - struct batch_request *req = NULL; - - if ((req = (batch_request *)calloc(1, sizeof(batch_request))) == NULL) - { - log_err(errno, __func__, msg_err_malloc); - } - else - { - - req->rq_type = type; - - req->rq_conn = -1; /* indicate not connected */ - req->rq_orgconn = -1; /* indicate not connected */ - req->rq_time = time(NULL); - req->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; - req->rq_noreply = FALSE; /* indicate reply is needed */ - } - - return(req); - } /* END alloc_br() */ - - - /* * close_quejob() * @@ -1139,241 +1103,15 @@ void close_quejob( void free_br( - struct batch_request *preq) + batch_request *preq) { if (preq == NULL) return; - if (preq->rq_id != NULL) - { - remove_batch_request(preq->rq_id); - free(preq->rq_id); - preq->rq_id = NULL; - } - - reply_free(&preq->rq_reply); - - if (preq->rq_extend) - { - free(preq->rq_extend); - preq->rq_extend = NULL; - } - - if (preq->rq_extra) - { - free(preq->rq_extra); - preq->rq_extra = NULL; - } - - switch (preq->rq_type) - { - case PBS_BATCH_QueueJob: - case PBS_BATCH_QueueJob2: - - free_attrlist(&preq->rq_ind.rq_queuejob.rq_attr); - - break; - - case PBS_BATCH_JobCred: - - if (preq->rq_ind.rq_jobcred.rq_data) - { - free(preq->rq_ind.rq_jobcred.rq_data); - preq->rq_ind.rq_jobcred.rq_data = NULL; - } - - break; - - case PBS_BATCH_MvJobFile: - case PBS_BATCH_jobscript: - case PBS_BATCH_jobscript2: - - if (preq->rq_ind.rq_jobfile.rq_data) - { - free(preq->rq_ind.rq_jobfile.rq_data); - preq->rq_ind.rq_jobfile.rq_data = NULL; - } - break; - - case PBS_BATCH_HoldJob: - - freebr_manage(&preq->rq_ind.rq_hold.rq_orig); - - break; - - case PBS_BATCH_CheckpointJob: - - freebr_manage(&preq->rq_ind.rq_manager); - - break; - - case PBS_BATCH_MessJob: - - if (preq->rq_ind.rq_message.rq_text) - { - free(preq->rq_ind.rq_message.rq_text); - preq->rq_ind.rq_message.rq_text = NULL; - } - - break; - - case PBS_BATCH_ModifyJob: - - case PBS_BATCH_AsyModifyJob: - - freebr_manage(&preq->rq_ind.rq_modify); - - break; - - case PBS_BATCH_StatusJob: - - case PBS_BATCH_StatusQue: - - case PBS_BATCH_StatusNode: - - case PBS_BATCH_StatusSvr: - /* DIAGTODO: handle PBS_BATCH_StatusDiag */ - - free_attrlist(&preq->rq_ind.rq_status.rq_attr); - - break; - - case PBS_BATCH_JobObit: - - free_attrlist(&preq->rq_ind.rq_jobobit.rq_attr); - - break; - - case PBS_BATCH_CopyFiles: - - case PBS_BATCH_DelFiles: - - freebr_cpyfile(&preq->rq_ind.rq_cpyfile); - - break; - - case PBS_BATCH_ModifyNode: - - case PBS_BATCH_Manager: - - freebr_manage(&preq->rq_ind.rq_manager); - - break; - - case PBS_BATCH_ReleaseJob: - - freebr_manage(&preq->rq_ind.rq_release); - - break; - - case PBS_BATCH_Rescq: - - free_rescrq(&preq->rq_ind.rq_rescq); - - break; - - case PBS_BATCH_SelectJobs: - - case PBS_BATCH_SelStat: - - free_attrlist(&preq->rq_ind.rq_select); - - break; - - case PBS_BATCH_SelStatAttr: - - free_attrlist(&preq->rq_ind.rq_select); - free_attrlist(&preq->rq_ind.rq_status.rq_attr); - - break; - - case PBS_BATCH_RunJob: - case PBS_BATCH_AsyrunJob: - - if (preq->rq_ind.rq_run.rq_destin) - { - free(preq->rq_ind.rq_run.rq_destin); - preq->rq_ind.rq_run.rq_destin = NULL; - } - break; - - default: - - /* NO-OP */ - - break; - } /* END switch (preq->rq_type) */ - - free(preq); - - return; + delete preq; } /* END free_br() */ - - -static void freebr_manage( - - struct rq_manage *pmgr) - - { - free_attrlist(&pmgr->rq_attr); - - return; - } /* END freebr_manage() */ - - - - -static void freebr_cpyfile( - - struct rq_cpyfile *pcf) - - { - - struct rqfpair *ppair; - - while ((ppair = (struct rqfpair *)GET_NEXT(pcf->rq_pair)) != NULL) - { - delete_link(&ppair->fp_link); - - if (ppair->fp_local != NULL) - free(ppair->fp_local); - - if (ppair->fp_rmt != NULL) - free(ppair->fp_rmt); - - free(ppair); - } - - return; - } /* END freebr_cpyfile() */ - - - - - -static void free_rescrq( - - struct rq_rescq *pq) - - { - int i; - - i = pq->rq_num; - - while (i--) - { - if (*(pq->rq_list + i) != NULL) - free(*(pq->rq_list + i)); - } - - if (pq->rq_list != NULL) - free(pq->rq_list); - - return; - } /* END free_rescrq() */ - /* END process_requests.c */ diff --git a/src/server/reply_send.c b/src/server/reply_send.c index d65d0a72e2..f20385c19c 100644 --- a/src/server/reply_send.c +++ b/src/server/reply_send.c @@ -168,7 +168,7 @@ int reply_send_svr( { /* Otherwise, the reply is to be sent to a remote client */ - if (request->rq_noreply != TRUE) + if (request->rq_noreply != true) { rc = dis_reply_write(sfds, &request->rq_reply); @@ -183,14 +183,6 @@ int reply_send_svr( } } - if (((request->rq_type != PBS_BATCH_AsyModifyJob) && - (request->rq_type != PBS_BATCH_AsyrunJob) && - (request->rq_type != PBS_BATCH_AsySignalJob)) || - (request->rq_noreply == TRUE)) - { - free_br(request); - } - return(rc); } /* END reply_send_svr() */ #endif /* PBS_MOM */ diff --git a/src/server/req_delete.c b/src/server/req_delete.c index 93d5acee81..e692854354 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -139,8 +139,6 @@ extern struct server server; extern int LOGLEVEL; extern all_jobs alljobs; -extern int issue_signal(job **, const char *, void (*)(batch_request *), void *, char *); - /* Private Functions in this file */ void post_delete_route(struct work_task *); @@ -313,7 +311,7 @@ void remove_stagein( "unable to remove staged in files for job"); } - free_br(preq); + delete preq; } return; @@ -479,7 +477,7 @@ int execute_job_delete( long force_cancel = FALSE; long status_cancel_queue = FALSE; - chk_job_req_permissions(&pjob,preq); + chk_job_req_permissions(&pjob, preq); if (pjob == NULL) { @@ -559,7 +557,7 @@ int execute_job_delete( log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf); - pwtnew = set_task(WORK_Timed,time_now + 1,post_delete_route,preq,FALSE); + pwtnew = set_task(WORK_Timed,time_now + 1, post_delete_route, new batch_request(*preq),FALSE); if (pwtnew == NULL) { @@ -659,7 +657,7 @@ int execute_job_delete( */ get_batch_request_id(preq); - if ((rc = issue_signal(&pjob, sigt, post_delete_mom1, strdup(del), strdup(preq->rq_id)))) + if ((rc = issue_signal(&pjob, sigt, post_delete_mom1, strdup(del), strdup(preq->rq_id.c_str())))) { /* cant send to MOM */ @@ -777,183 +775,6 @@ int execute_job_delete( -int copy_attribute_list( - - batch_request *preq, - batch_request *preq_tmp) - - { - svrattrl *pal = (svrattrl *)GET_NEXT(preq->rq_ind.rq_manager.rq_attr); - tlist_head *phead = &preq_tmp->rq_ind.rq_manager.rq_attr; - svrattrl *newpal = NULL; - - while (pal != NULL) - { - newpal = (svrattrl *)calloc(1, pal->al_tsize + 1); - if (!newpal) - { - free_br(preq_tmp); - return(PBSE_SYSTEM); - } - - CLEAR_LINK(newpal->al_link); - - newpal->al_atopl.next = 0; - newpal->al_tsize = pal->al_tsize + 1; - newpal->al_nameln = pal->al_nameln; - newpal->al_flags = pal->al_flags; - newpal->al_atopl.name = (char *)newpal + sizeof(svrattrl); - strcpy((char *)newpal->al_atopl.name, pal->al_atopl.name); - newpal->al_nameln = pal->al_nameln; - newpal->al_atopl.resource = newpal->al_atopl.name + newpal->al_nameln; - - if (pal->al_atopl.resource != NULL) - strcpy((char *)newpal->al_atopl.resource, pal->al_atopl.resource); - - newpal->al_rescln = pal->al_rescln; - newpal->al_atopl.value = newpal->al_atopl.name + newpal->al_nameln + newpal->al_rescln; - strcpy((char *)newpal->al_atopl.value, pal->al_atopl.value); - newpal->al_valln = pal->al_valln; - newpal->al_atopl.op = pal->al_atopl.op; - - pal = (struct svrattrl *)GET_NEXT(pal->al_link); - } - - if ((phead != NULL) && - (newpal != NULL)) - append_link(phead, &newpal->al_link, newpal); - - return(PBSE_NONE); - } /* END copy_attribute_list() */ - - - - -/* - * duplicate_request() - * duplicates preq and returns the duplicate request - * @param preq - the request to duplicate - * @param job_index - if desired, replace the job id with the sub job id. - * The sub-job has the index job_index and this is only performed if this - * value isn't -1 - */ - -batch_request *duplicate_request( - - batch_request *preq, /* I */ - int job_index) /* I - optional */ - - { - batch_request *preq_tmp = alloc_br(preq->rq_type); - char *ptr1; - char *ptr2; - char newjobname[PBS_MAXSVRJOBID+1]; - - if (preq_tmp == NULL) - return(NULL); - - preq_tmp->rq_perm = preq->rq_perm; - preq_tmp->rq_fromsvr = preq->rq_fromsvr; - preq_tmp->rq_conn = preq->rq_conn; - preq_tmp->rq_time = preq->rq_time; - preq_tmp->rq_orgconn = preq->rq_orgconn; - - memcpy(preq_tmp->rq_ind.rq_manager.rq_objname, - preq->rq_ind.rq_manager.rq_objname, PBS_MAXSVRJOBID + 1); - - strcpy(preq_tmp->rq_user, preq->rq_user); - strcpy(preq_tmp->rq_host, preq->rq_host); - - if (preq->rq_extend != NULL) - preq_tmp->rq_extend = strdup(preq->rq_extend); - - switch (preq->rq_type) - { - /* This function was created for a modify array request (PBS_BATCH_ModifyJob) - the preq->rq_ind structure was allocated in dis_request_read. If other - BATCH types are needed refer to that function to see how the rq_ind structure - was allocated and then copy it here. */ - case PBS_BATCH_DeleteJob: - case PBS_BATCH_HoldJob: - case PBS_BATCH_CheckpointJob: - case PBS_BATCH_ModifyJob: - case PBS_BATCH_AsyModifyJob: - - /* based on how decode_DIS_Manage allocates data */ - CLEAR_HEAD(preq_tmp->rq_ind.rq_manager.rq_attr); - - preq_tmp->rq_ind.rq_manager.rq_cmd = preq->rq_ind.rq_manager.rq_cmd; - preq_tmp->rq_ind.rq_manager.rq_objtype = preq->rq_ind.rq_manager.rq_objtype; - - if (job_index != -1) - { - /* If this is a job array it is possible we only have the array name - and not the individual job. We need to find out what we have and - modify the name if needed */ - ptr1 = strstr(preq->rq_ind.rq_manager.rq_objname, "[]"); - if (ptr1) - { - ptr1++; - strcpy(newjobname, preq->rq_ind.rq_manager.rq_objname); - - if ((ptr2 = strstr(newjobname, "[]")) != NULL) - { - ptr2++; - *ptr2 = 0; - } - - sprintf(preq_tmp->rq_ind.rq_manager.rq_objname,"%s%d%s", - newjobname, job_index, ptr1); - } - else - strcpy(preq_tmp->rq_ind.rq_manager.rq_objname, preq->rq_ind.rq_manager.rq_objname); - } - - /* copy the attribute list */ - if (copy_attribute_list(preq, preq_tmp) != PBSE_NONE) - return(NULL); - - break; - - case PBS_BATCH_SignalJob: - - strcpy(preq_tmp->rq_ind.rq_signal.rq_jid, preq->rq_ind.rq_signal.rq_jid); - strcpy(preq_tmp->rq_ind.rq_signal.rq_signame, preq->rq_ind.rq_signal.rq_signame); - preq_tmp->rq_extra = strdup((char *)preq->rq_extra); - - break; - - case PBS_BATCH_MessJob: - - strcpy(preq_tmp->rq_ind.rq_message.rq_jid, preq->rq_ind.rq_message.rq_jid); - preq_tmp->rq_ind.rq_message.rq_file = preq->rq_ind.rq_message.rq_file; - strcpy(preq_tmp->rq_ind.rq_message.rq_text, preq->rq_ind.rq_message.rq_text); - - break; - - case PBS_BATCH_RunJob: - case PBS_BATCH_AsyrunJob: - - if (preq->rq_ind.rq_run.rq_destin) - preq_tmp->rq_ind.rq_run.rq_destin = strdup(preq->rq_ind.rq_run.rq_destin); - - break; - - case PBS_BATCH_Rerun: - - strcpy(preq_tmp->rq_ind.rq_rerun, preq->rq_ind.rq_rerun); - break; - - default: - - break; - } - - return(preq_tmp); - } /* END duplicate_request() */ - - - void *delete_all_work( void *vp) @@ -967,7 +788,7 @@ void *delete_all_work( return(NULL); } - batch_request *preq_dup = duplicate_request(preq); + batch_request preq_dup(*preq); job *pjob; all_jobs_iterator *iter = NULL; int failed_deletes = 0; @@ -1008,7 +829,7 @@ void *delete_all_work( // use mutex manager to make sure job mutex locks are properly handled at exit mutex_mgr job_mutex(pjob->ji_mutex, true); - if ((rc = forced_jobpurge(pjob, preq_dup)) == PURGE_SUCCESS) + if ((rc = forced_jobpurge(pjob, &preq_dup)) == PURGE_SUCCESS) { job_mutex.set_unlock_on_exit(false); @@ -1019,12 +840,6 @@ void *delete_all_work( { job_mutex.unlock(); - if (rc == -1) - { - //forced_jobpurge freed preq_dup so reallocate it. - preq_dup = duplicate_request(preq); - preq_dup->rq_noreply = TRUE; - } continue; } @@ -1033,29 +848,17 @@ void *delete_all_work( /* mutex is freed below */ if (rc == PBSE_NONE) { - if ((rc = execute_job_delete(pjob, Msg, preq_dup)) == PBSE_NONE) + if ((rc = execute_job_delete(pjob, Msg, &preq_dup)) == PBSE_NONE) { // execute_job_delete() handles mutex so don't unlock on exit job_mutex.set_unlock_on_exit(false); - reply_ack(preq_dup); + reply_ack(&preq_dup); } - /* preq_dup has been freed at this point. Either reallocate it or set it to NULL*/ - if (rc == PURGE_SUCCESS) - { - preq_dup = duplicate_request(preq); - preq_dup->rq_noreply = TRUE; - } - else - preq_dup = NULL; } if (rc != PURGE_SUCCESS) { - /* duplicate the preq so we don't have a problem with double frees */ - preq_dup = duplicate_request(preq); - preq_dup->rq_noreply = TRUE; - if ((rc == MOM_DELETE) || (rc == ROUTE_DELETE)) failed_deletes++; @@ -1069,14 +872,6 @@ void *delete_all_work( if (failed_deletes == 0) { reply_ack(preq); - - /* PURGE SUCCESS means this was qdel -p all. In this case no reply_*() - * functions have been called */ - if (rc == PURGE_SUCCESS) - { - free_br(preq_dup); - preq_dup = NULL; - } } else { @@ -1087,11 +882,6 @@ void *delete_all_work( req_reject(PBSE_SYSTEM, 0, preq, NULL, tmpLine); } - /* preq_dup happens at the end of the loop, so free the extra one if - * it is there */ - if (preq_dup != NULL) - free_br(preq_dup); - return(NULL); } /* END delete_all_work() */ @@ -1109,7 +899,7 @@ int handle_delete_all( if (preq_tmp != NULL) { reply_ack(preq_tmp); - preq->rq_noreply = TRUE; /* set for no more replies */ + preq->rq_noreply = true; /* set for no more replies */ enqueue_threadpool_request(delete_all_work, preq, request_pool); } else @@ -1178,7 +968,7 @@ int handle_single_delete( if (preq_tmp != NULL) { reply_ack(preq_tmp); - preq->rq_noreply = TRUE; /* set for no more replies */ + preq->rq_noreply = true; /* set for no more replies */ enqueue_threadpool_request(single_delete_work, preq, async_pool); } else @@ -1279,7 +1069,7 @@ int req_deletejob( snprintf(log_buf,sizeof(log_buf), "Deleting job asynchronously"); log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,preq->rq_ind.rq_delete.rq_objname,log_buf); - preq_tmp = duplicate_request(preq); + preq_tmp = new batch_request(*preq); } } @@ -1359,10 +1149,13 @@ void post_delete_route( struct work_task *pwt) { - batch_request *preq = get_remove_batch_request((char *)pwt->wt_parm1); + batch_request *preq = (batch_request *)pwt->wt_parm1; if (preq != NULL) + { req_deletejob(preq); + delete preq; + } free(pwt->wt_mutex); free(pwt); @@ -1404,7 +1197,6 @@ void post_delete_mom1( { preq_clt = get_remove_batch_request(preq_sig->rq_extend); } - free_br(preq_sig); /* the client request has been handled another way, nothing left to do */ if (preq_clt == NULL) @@ -1529,7 +1321,7 @@ void post_delete_mom2( if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) { - issue_signal(&pjob, sigk, free_br, NULL, NULL); + issue_signal(&pjob, sigk, NULL, NULL, NULL); if (pjob != NULL) { @@ -1712,7 +1504,7 @@ void job_delete_nanny( log_err(-1, "job nanny", log_buf); /* build up a Signal Job batch request */ - if ((newreq = alloc_br(PBS_BATCH_SignalJob)) != NULL) + if ((newreq = new batch_request(PBS_BATCH_SignalJob)) != NULL) { strcpy(newreq->rq_ind.rq_signal.rq_jid, pjob->ji_qs.ji_jobid); snprintf(newreq->rq_ind.rq_signal.rq_signame, sizeof(newreq->rq_ind.rq_signal.rq_signame), "%s", sigk); @@ -1769,7 +1561,6 @@ void post_job_delete_nanny( if (!nanny) { /* the admin disabled nanny within the last minute or so */ - free_br(preq_sig); return; } @@ -1783,8 +1574,6 @@ void post_job_delete_nanny( log_event(PBSEVENT_ERROR,PBS_EVENTCLASS_JOB,preq_sig->rq_ind.rq_signal.rq_jid,log_buf); - free_br(preq_sig); - return; } @@ -1799,9 +1588,7 @@ void post_job_delete_nanny( free_nodes(pjob); set_resc_assigned(pjob, DECR); - - free_br(preq_sig); - + job_mutex.set_unlock_on_exit(false); svr_job_purge(pjob); @@ -1809,9 +1596,6 @@ void post_job_delete_nanny( return; } - /* free task */ - free_br(preq_sig); - return; } /* END post_job_delete_nanny() */ diff --git a/src/server/req_deletearray.c b/src/server/req_deletearray.c index 6f12214bcb..e8f65e14e3 100644 --- a/src/server/req_deletearray.c +++ b/src/server/req_deletearray.c @@ -35,7 +35,6 @@ extern int has_job_delete_nanny(struct job *); extern void setup_apply_job_delete_nanny(struct job *, time_t time_now); extern void remove_stagein(job **pjob); extern void change_restart_comment_if_needed(struct job *); -int issue_signal(job **, const char *, void(*)(batch_request *), void *, char *); extern char *msg_unkarrayid; extern char *msg_permlog; @@ -108,7 +107,7 @@ bool attempt_delete( /* need to issue a signal to the mom, but we don't want to sent an ack to the * client when the mom replies */ - issue_signal(&pjob, "SIGTERM", free_br, NULL, NULL); + issue_signal(&pjob, "SIGTERM", NULL, NULL, NULL); } if (pjob != NULL) diff --git a/src/server/req_holdarray.c b/src/server/req_holdarray.c index 01ec2e73d8..8358887993 100644 --- a/src/server/req_holdarray.c +++ b/src/server/req_holdarray.c @@ -61,7 +61,6 @@ void hold_job( (csv_find_string(pattr->at_val.at_str, "enabled") != NULL)))) { /* TODO */ - /* preq_tmp = alloc_br(preq->rq_type); */ } else if (old_hold != *hold_val) diff --git a/src/server/req_holdjob.c b/src/server/req_holdjob.c index 913ec14e29..8d271897a2 100644 --- a/src/server/req_holdjob.c +++ b/src/server/req_holdjob.c @@ -177,7 +177,6 @@ int req_holdjob( pbs_attribute *pattr; batch_request *preq = (struct batch_request *)vp; char log_buf[LOCAL_LOG_BUF_SIZE]; - batch_request *dup_req = NULL; pjob = chk_job_request(preq->rq_ind.rq_hold.rq_orig.rq_objname, preq); @@ -236,15 +235,11 @@ int req_holdjob( snprintf(preq->rq_ind.rq_hold.rq_orig.rq_objname, sizeof(preq->rq_ind.rq_hold.rq_orig.rq_objname), "%s", pjob->ji_qs.ji_jobid); - if ((dup_req = duplicate_request(preq)) == NULL) - { - req_reject(rc, 0, preq, NULL, "memory allocation failure"); - } - /* The dup_req is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - else if ((rc = relay_to_mom(&pjob, dup_req, NULL)) != PBSE_NONE) + + batch_request dup_req(*preq); + + if ((rc = relay_to_mom(&pjob, &dup_req, NULL)) != PBSE_NONE) { - free_br(dup_req); *hold_val = old_hold; /* reset to the old value */ req_reject(rc, 0, preq, NULL, "relay to mom failed"); @@ -270,7 +265,7 @@ int req_holdjob( else job_mutex.set_unlock_on_exit(false); - process_hold_reply(dup_req); + process_hold_reply(&dup_req); } } #ifdef ENABLE_BLCR @@ -326,7 +321,6 @@ void *req_checkpointjob( int rc; pbs_attribute *pattr; char log_buf[LOCAL_LOG_BUF_SIZE]; - batch_request *dup_req = NULL; if ((pjob = chk_job_request(preq->rq_ind.rq_manager.rq_objname, preq)) == NULL) { @@ -345,17 +339,11 @@ void *req_checkpointjob( { /* have MOM attempt checkpointing */ - if ((dup_req = duplicate_request(preq)) == NULL) - { - req_reject(PBSE_SYSTEM, 0, preq, NULL, "failure to allocate memory"); - } + batch_request dup_req(*preq); - /* The dup_req is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - else if ((rc = relay_to_mom(&pjob, dup_req, NULL)) != PBSE_NONE) + if ((rc = relay_to_mom(&pjob, &dup_req, NULL)) != PBSE_NONE) { req_reject(rc, 0, preq, NULL, NULL); - free_br(dup_req); if (pjob == NULL) job_mutex.set_unlock_on_exit(false); @@ -374,13 +362,12 @@ void *req_checkpointjob( else job_mutex.set_unlock_on_exit(false); - process_checkpoint_reply(dup_req); + process_checkpoint_reply(&dup_req); } } else { - /* Job does not have checkpointing enabled, so reject the request */ - + // Job does not have checkpointing enabled, so reject the request log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf); req_reject(PBSE_IVALREQ, 0, preq, NULL, "job is not checkpointable"); @@ -391,7 +378,6 @@ void *req_checkpointjob( - /* * release_job - releases the hold on job j * @param j - the job to modify diff --git a/src/server/req_jobobit.c b/src/server/req_jobobit.c index cde783afb0..9146215a0f 100644 --- a/src/server/req_jobobit.c +++ b/src/server/req_jobobit.c @@ -225,7 +225,7 @@ struct batch_request *setup_cpyfiles( { /* allocate and initialize the batch request struct */ - preq = alloc_br(PBS_BATCH_CopyFiles); + preq = new batch_request(PBS_BATCH_CopyFiles); if (preq == NULL) { @@ -286,7 +286,7 @@ struct batch_request *setup_cpyfiles( { /* FAILURE */ - free_br(preq); + delete preq; log_event( PBSEVENT_ERROR | PBSEVENT_JOB, @@ -383,7 +383,7 @@ struct batch_request *return_stdfile( if (preq == NULL) { - preq = alloc_br(PBS_BATCH_ReturnFiles); + preq = new batch_request(PBS_BATCH_ReturnFiles); } strcpy(preq->rq_ind.rq_returnfiles.rq_jobid, pjob->ji_qs.ji_jobid); @@ -430,7 +430,7 @@ struct batch_request *cpy_stdfile( { /* the job is interactive, don't bother to return output file */ if (preq != NULL) - free_br(preq); + delete preq; return(NULL); } @@ -461,7 +461,7 @@ struct batch_request *cpy_stdfile( log_buf); if (preq != NULL) - free_br(preq); + delete preq; return(NULL); } @@ -1078,8 +1078,7 @@ int handle_returnstd( * "faking" the immediate work task and falling through to * the next case. */ - - free_br(preq); + delete preq; } if ((pjob = svr_find_job(job_id, TRUE)) != NULL) @@ -1339,7 +1338,7 @@ int handle_stageout( preq->rq_extra = NULL; } - free_br(preq); + delete preq; preq = NULL; } /* END if preq != NULL */ @@ -1356,7 +1355,7 @@ int handle_stageout( if (preq != NULL) { - free_br(preq); + delete preq; } if (job_momname != NULL) @@ -1490,7 +1489,7 @@ int handle_stagedel( unlock_ji_mutex(pjob, __func__, "5", LOGLEVEL); } - free_br(preq); + delete preq; } if ((pjob = svr_find_job(job_id, TRUE)) != NULL) @@ -1532,7 +1531,7 @@ int handle_exited( } /* tell mom to delete the job, send final track and purge it */ - preq = alloc_br(PBS_BATCH_DeleteJob); + preq = new batch_request(PBS_BATCH_DeleteJob); if (preq != NULL) { @@ -1540,7 +1539,7 @@ int handle_exited( if ((handle = mom_comm(pjob, on_job_exit_task)) < 0) { - free_br(preq); + delete preq; return(PBSE_CONNECT); } else @@ -1559,7 +1558,7 @@ int handle_exited( } } - free_br(preq); + delete preq; } else job_mutex.unlock(); @@ -2333,7 +2332,7 @@ void on_job_rerun( /* ask mom to send them back to the server */ /* mom deletes her copy if returned ok */ - if ((preq = alloc_br(PBS_BATCH_Rerun)) == NULL) + if ((preq = new batch_request(PBS_BATCH_Rerun)) == NULL) return; strcpy(preq->rq_ind.rq_rerun, pjob->ji_qs.ji_jobid); @@ -2623,7 +2622,7 @@ void on_job_rerun( case JOB_SUBSTATE_RERUN3: /* need to have MOM delete her copy of the job */ - preq = alloc_br(PBS_BATCH_DeleteJob); + preq = new batch_request(PBS_BATCH_DeleteJob); if (preq != NULL) { @@ -3677,7 +3676,7 @@ int req_jobobit( "No resources used found"); } - tmppreq = alloc_br(PBS_BATCH_JobObit); + tmppreq = new batch_request(PBS_BATCH_JobObit); if (tmppreq == NULL) { diff --git a/src/server/req_manager.c b/src/server/req_manager.c index 505f515e94..8c5c8b28ce 100644 --- a/src/server/req_manager.c +++ b/src/server/req_manager.c @@ -2027,8 +2027,8 @@ static bool requeue_or_delete_jobs( if(pjob != NULL) { char *dup_jobid = strdup(pjob->ji_qs.ji_jobid); - batch_request *brRerun = alloc_br(PBS_BATCH_Rerun); - batch_request *brDelete = alloc_br(PBS_BATCH_DeleteJob); + batch_request *brRerun = new batch_request(PBS_BATCH_Rerun); + batch_request *brDelete = new batch_request(PBS_BATCH_DeleteJob); if((brRerun == NULL)||(brDelete == NULL)) { free_br(brRerun); diff --git a/src/server/req_message.c b/src/server/req_message.c index c3142b319c..37dee30210 100644 --- a/src/server/req_message.c +++ b/src/server/req_message.c @@ -128,7 +128,6 @@ void *req_messagejob( { job *pjob; int rc; - batch_request *dup_req = NULL; if ((pjob = chk_job_request(preq->rq_ind.rq_message.rq_jid, preq)) == NULL) return(NULL); @@ -144,22 +143,15 @@ void *req_messagejob( return(NULL); } - if ((dup_req = duplicate_request(preq)) == NULL) - { - req_reject(PBSE_MEM_MALLOC, 0, preq, NULL, NULL); - } - /* pass the request on to MOM */ - /* The dup_req is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - else if ((rc = relay_to_mom(&pjob, dup_req, NULL)) != PBSE_NONE) + batch_request dup_req(*preq); + + if ((rc = relay_to_mom(&pjob, &dup_req, NULL)) != PBSE_NONE) { req_reject(rc, 0, preq, NULL, NULL); /* unable to get to MOM */ - free_br(dup_req); } else { - post_message_req(dup_req); - free_br(preq); + post_message_req(&dup_req); } /* After MOM acts and replies to us, we pick up in post_message_req() */ diff --git a/src/server/req_modify.c b/src/server/req_modify.c index c00fa8aba8..b563874720 100644 --- a/src/server/req_modify.c +++ b/src/server/req_modify.c @@ -230,7 +230,6 @@ void mom_cleanup_checkpoint_hold( job *pjob; char *jobid; - batch_request *preq; char log_buf[LOCAL_LOG_BUF_SIZE]; time_t time_now = time(NULL); @@ -279,44 +278,30 @@ void mom_cleanup_checkpoint_hold( if (pjob->ji_qs.ji_state != JOB_STATE_RUNNING) { - if ((preq = alloc_br(PBS_BATCH_DeleteJob)) == NULL) - { - log_err(-1, __func__, "unable to allocate DeleteJob request - big trouble!"); - } - else + batch_request preq(PBS_BATCH_DeleteJob); + + strcpy(preq.rq_ind.rq_delete.rq_objname, pjob->ji_qs.ji_jobid); + + if ((rc = relay_to_mom(&pjob, &preq, NULL)) != PBSE_NONE) { - strcpy(preq->rq_ind.rq_delete.rq_objname, pjob->ji_qs.ji_jobid); - /* The preq is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - if ((rc = relay_to_mom(&pjob, preq, NULL)) != PBSE_NONE) + if (pjob != NULL) { - if (pjob != NULL) - { - snprintf(log_buf,sizeof(log_buf), - "Unable to relay information to mom for job '%s'\n", - pjob->ji_qs.ji_jobid); - - log_err(rc, __func__, log_buf); - } - else - job_mutex.set_unlock_on_exit(false); - - free_br(preq); - - return; + snprintf(log_buf,sizeof(log_buf), + "Unable to relay information to mom for job '%s'\n", + pjob->ji_qs.ji_jobid); + + log_err(rc, __func__, log_buf); } else - free_br(preq); + job_mutex.set_unlock_on_exit(false); - if ((LOGLEVEL >= 7) && - (pjob != NULL)) - { - log_event( - PBSEVENT_JOB, - PBS_EVENTCLASS_JOB, - pjob->ji_qs.ji_jobid, - "requested mom cleanup"); - } + return; + } + + if ((LOGLEVEL >= 7) && + (pjob != NULL)) + { + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, "requested mom cleanup"); } } else @@ -357,8 +342,6 @@ void chkpt_xfr_hold( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf); } - - free_br(preq); set_task(WORK_Immed, 0, mom_cleanup_checkpoint_hold, strdup(pjob->ji_qs.ji_jobid), FALSE); @@ -367,22 +350,6 @@ void chkpt_xfr_hold( - - -/* - * chkpt_xfr_done - Handle the clean up of the transfer of the checkpoint files. - */ - -void chkpt_xfr_done( - - batch_request *preq) - - { - free_br(preq); - } /* END chkpt_xfr_done() */ - - - /* * has_unalterable_attribute() * @@ -691,7 +658,7 @@ int modify_job( if (copy_checkpoint_files) { - struct batch_request *momreq = 0; + struct batch_request *momreq = NULL; momreq = cpy_checkpoint(momreq, pjob, JOB_ATR_checkpoint_name, CKPT_DIR_OUT); if (momreq != NULL) @@ -699,14 +666,10 @@ int modify_job( /* have files to copy */ momreq->rq_extra = strdup(pjob->ji_qs.ji_jobid); - /* The momreq is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ rc = relay_to_mom(&pjob, momreq, NULL); if (rc != PBSE_NONE) { - free_br(momreq); - if (pjob != NULL) { snprintf(log_buf,sizeof(log_buf), @@ -720,8 +683,8 @@ int modify_job( } else if (checkpoint_req == CHK_HOLD) chkpt_xfr_hold(momreq, pjob); - else - chkpt_xfr_done(momreq); + + delete momreq; } else { @@ -767,10 +730,11 @@ int modify_whole_array( else { /* NO_MOM_RELAY will prevent modify_job from calling relay_to_mom */ - batch_request *array_req = duplicate_request(preq, i); + batch_request *array_req = new batch_request(*preq); + array_req->update_object_id(i); mutex_mgr job_mutex(pjob->ji_mutex, true); pthread_mutex_unlock(pa->ai_mutex); - array_req->rq_noreply = TRUE; + array_req->rq_noreply = true; rc = modify_job((void **)&pjob, plist, array_req, checkpoint_req, NO_MOM_RELAY); if (rc != PBSE_NONE) { @@ -936,7 +900,7 @@ void *req_modifyarray( /* If async modify, reply now; otherwise reply is handled later */ if (preq->rq_type == PBS_BATCH_AsyModifyJob) { - preq->rq_noreply = TRUE; /* set for no more replies */ + preq->rq_noreply = true; /* set for no more replies */ reply_ack(preq); enqueue_threadpool_request(modify_array_work, preq, async_pool); @@ -949,22 +913,21 @@ void *req_modifyarray( -void *modify_job_work( +int modify_job_work( - batch_request *vp) /* I */ + batch_request *preq) /* I */ { job *pjob; svrattrl *plist; int checkpoint_req = FALSE; - batch_request *preq = (struct batch_request *)vp; pjob = svr_find_job(preq->rq_ind.rq_modify.rq_objname, FALSE); if (pjob == NULL) { req_reject(PBSE_JOBNOTFOUND, 0, preq, NULL, "Job unexpectedly deleted"); - return(NULL); + return(PBSE_JOBNOTFOUND); } mutex_mgr job_mutex(pjob->ji_mutex, true); @@ -987,12 +950,11 @@ void *modify_job_work( /* modify_job will free preq and respond to it */ modify_job((void **)&pjob, plist, preq, checkpoint_req, 0); - return(NULL); + return(PBSE_NONE); } /* END modify_job_work() */ - /* * req_modifyjob - service the Modify Job Request * @@ -1008,7 +970,6 @@ void *req_modifyjob( { job *pjob; svrattrl *plist; - char log_buf[LOCAL_LOG_BUF_SIZE]; char *correct_jobname_p; pjob = chk_job_request(preq->rq_ind.rq_modify.rq_objname, preq); @@ -1059,18 +1020,12 @@ void *req_modifyjob( /* reply_ack will free preq. We need to copy it before we call reply_ack */ batch_request *new_preq; - new_preq = duplicate_request(preq, -1); - if (new_preq == NULL) - { - sprintf(log_buf, "failed to duplicate batch request"); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - return(NULL); - } + new_preq = new batch_request(*preq); get_batch_request_id(new_preq); reply_ack(preq); - new_preq->rq_noreply = TRUE; /* set for no more replies */ + new_preq->rq_noreply = true; /* set for no more replies */ enqueue_threadpool_request((void *(*)(void *))modify_job_work, preq, async_pool); } @@ -1082,8 +1037,6 @@ void *req_modifyjob( - - /* * modify_job_attr - modify the attributes of a job atomically * Used by req_modifyjob() to alter the job attributes and by @@ -1354,6 +1307,8 @@ int modify_job_attr( /* * post_modify_arrayreq - clean up after sending modify request to MOM + * + * FIXME: This appears to be dead code */ void post_modify_arrayreq( diff --git a/src/server/req_quejob.c b/src/server/req_quejob.c index 3f3b9742d0..12595c8470 100644 --- a/src/server/req_quejob.c +++ b/src/server/req_quejob.c @@ -1636,12 +1636,12 @@ int perform_commit_work( if ((pattr->at_val.at_long == 0) && (nodes_avail > 0)) { /* Create a new batch request and fill it in */ - preq_run = alloc_br(PBS_BATCH_RunJob); + preq_run = new batch_request(PBS_BATCH_RunJob); preq_run->rq_perm = preq->rq_perm | ATR_DFLAG_OPWR; preq_run->rq_ind.rq_run.rq_resch = 0; preq_run->rq_ind.rq_run.rq_destin = rq_destin; preq_run->rq_fromsvr = preq->rq_fromsvr; - preq_run->rq_noreply = TRUE; /* set for no replies */ + preq_run->rq_noreply = true; /* set for no replies */ strcpy(preq_run->rq_user, preq->rq_user); strcpy(preq_run->rq_host, preq->rq_host); strcpy(preq_run->rq_ind.rq_run.rq_jid, preq->rq_ind.rq_rdytocommit); @@ -2532,12 +2532,12 @@ int req_commit2( if ((pattr->at_val.at_long == 0) && (nodes_avail > 0)) { /* Create a new batch request and fill it in */ - preq_run = alloc_br(PBS_BATCH_RunJob); + preq_run = new batch_request(PBS_BATCH_RunJob); preq_run->rq_perm = preq->rq_perm | ATR_DFLAG_OPWR; preq_run->rq_ind.rq_run.rq_resch = 0; preq_run->rq_ind.rq_run.rq_destin = rq_destin; preq_run->rq_fromsvr = preq->rq_fromsvr; - preq_run->rq_noreply = TRUE; /* set for no replies */ + preq_run->rq_noreply = true; /* set for no replies */ strcpy(preq_run->rq_user, preq->rq_user); strcpy(preq_run->rq_host, preq->rq_host); strcpy(preq_run->rq_ind.rq_run.rq_jid, preq->rq_ind.rq_rdytocommit); diff --git a/src/server/req_register.c b/src/server/req_register.c index 6dfd840f1f..40d57b1ed6 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -1335,8 +1335,6 @@ void post_doq( } } - free_br(preq); - return; } /* END post_doq() */ @@ -1380,7 +1378,7 @@ int alter_unreg( if ((pnewd == 0) || (find_dependjob(pnewd, oldjd->dc_child.c_str()) == 0)) { - int rc = send_depend_req(pjob, oldjd, type, JOB_DEPEND_OP_UNREG, SYNC_SCHED_HINT_NULL, free_br,false); + int rc = send_depend_req(pjob, oldjd, type, JOB_DEPEND_OP_UNREG, SYNC_SCHED_HINT_NULL, NULL, false); if (rc == PBSE_JOBNOTFOUND) return(rc); @@ -1498,7 +1496,7 @@ int depend_on_que( for (unsigned int i = 0; i < pparent.size(); i++) { if ((rc = send_depend_req(pjob, pparent[i], type, JOB_DEPEND_OP_REGISTER, - SYNC_SCHED_HINT_NULL, post_doq,false)) != PBSE_NONE) + SYNC_SCHED_HINT_NULL, post_doq, false)) != PBSE_NONE) break; } @@ -1564,8 +1562,6 @@ void post_doe( } } - free_br(preq); - return; } /* END post_doe() */ @@ -1606,7 +1602,8 @@ int depend_on_exec( pdep->dp_type, JOB_DEPEND_OP_RELEASE, SYNC_SCHED_HINT_NULL, - post_doe,false) == PBSE_JOBNOTFOUND) + post_doe, + false) == PBSE_JOBNOTFOUND) { return(PBSE_JOBNOTFOUND); } @@ -1631,7 +1628,8 @@ int depend_on_exec( pdep->dp_type, JOB_DEPEND_OP_READY, SYNC_SCHED_HINT_NULL, - free_br,false) == PBSE_JOBNOTFOUND) + NULL, + false) == PBSE_JOBNOTFOUND) { return(PBSE_JOBNOTFOUND); } @@ -1765,7 +1763,7 @@ int depend_on_term( { pparent = pdep->dp_jobs[i]; - rc = send_depend_req(pjob, pparent, type, JOB_DEPEND_OP_DELETE, SYNC_SCHED_HINT_NULL, free_br,true); + rc = send_depend_req(pjob, pparent, type, JOB_DEPEND_OP_DELETE, SYNC_SCHED_HINT_NULL, NULL, true); if (rc == PBSE_JOBNOTFOUND) { @@ -1786,7 +1784,7 @@ int depend_on_term( pparent = pdep->dp_jobs[i]; /* "release" the job to execute */ - if ((rc = send_depend_req(pjob, pparent, type, op, SYNC_SCHED_HINT_NULL, free_br,true)) != PBSE_NONE) + if ((rc = send_depend_req(pjob, pparent, type, op, SYNC_SCHED_HINT_NULL, NULL, true)) != PBSE_NONE) { return(rc); } @@ -1847,7 +1845,7 @@ int release_cheapest( hint = SYNC_SCHED_HINT_FIRST; if ((rc = send_depend_req(pjob, cheapest, JOB_DEPEND_TYPE_SYNCWITH, - JOB_DEPEND_OP_RELEASE, hint, free_br,false)) == PBSE_NONE) + JOB_DEPEND_OP_RELEASE, hint, NULL, false)) == PBSE_NONE) { cheapest->dc_state = JOB_DEPEND_OP_RELEASE; } @@ -2427,27 +2425,17 @@ int send_depend_req( bool bAsyncOk) { - int rc = 0; - int i; - char job_id[PBS_MAXSVRJOBID + 1]; - char br_id[MAXLINE]; - - struct batch_request *preq; - char log_buf[LOCAL_LOG_BUF_SIZE]; + int rc = 0; + int i; + char job_id[PBS_MAXSVRJOBID + 1]; - preq = alloc_br(PBS_BATCH_RegistDep); - - if (preq == NULL) - { - log_err(errno, __func__, msg_err_malloc); - return(PBSE_SYSTEM); - } + batch_request *preq = new batch_request(PBS_BATCH_RegistDep); + char log_buf[LOCAL_LOG_BUF_SIZE]; for (i = 0;i < PBS_MAXUSER;++i) { if (pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str == NULL) { - free_br(preq); return(PBSE_BADATVAL); } @@ -2501,9 +2489,6 @@ int send_depend_req( strcpy(job_id, pjob->ji_qs.ji_jobid); unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL); - get_batch_request_id(preq); - snprintf(br_id, sizeof(br_id), "%s", preq->rq_id); - if (bAsyncOk) { snprintf(preq->rq_host, sizeof(preq->rq_host), "%s", server_name); @@ -2512,23 +2497,14 @@ int send_depend_req( } else { - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); } if (rc != PBSE_NONE) { - /* requests are conditionally freed in issue_to_svr() */ - if (preq != NULL) - { - free_br(preq); - } - sprintf(log_buf, "Unable to perform dependency with job %s\n", pparent->dc_child.c_str()); log_err(rc, __func__, log_buf); - if ((preq = get_remove_batch_request(br_id)) != NULL) - free_br(preq); - if ((pjob = svr_find_job(job_id, TRUE)) == NULL) { return(PBSE_JOBNOTFOUND); @@ -2538,9 +2514,13 @@ int send_depend_req( } /* local requests have already been processed and freed. Do not attempt to * free or reference again. */ - else if (preq != NULL) + else if ((preq != NULL) && + (postfunc != NULL)) postfunc(preq); + if (preq != NULL) + delete preq; + if ((pjob = svr_find_job(job_id, TRUE)) == NULL) { return(PBSE_JOBNOTFOUND); diff --git a/src/server/req_rerun.c b/src/server/req_rerun.c index 670ed195fd..ed3ab11909 100644 --- a/src/server/req_rerun.c +++ b/src/server/req_rerun.c @@ -118,7 +118,6 @@ extern char *msg_jobrerun; extern void rel_resc(job *); extern job *chk_job_request(char *, struct batch_request *); -extern int issue_signal(job **, const char *, void(*)(batch_request *), void *, char *); int finalize_rerunjob(struct batch_request *preq,job *pjob,int rc); @@ -160,7 +159,6 @@ void delay_and_send_sig_kill( { preq_clt = get_remove_batch_request(preq_sig->rq_extend); } - free_br(preq_sig); /* the client request has been handled another way, nothing left to do */ if (preq_clt == NULL) @@ -316,8 +314,6 @@ void post_rerun( } } - free_br(preq); - return; } /* END post_rerun() */ @@ -533,25 +529,32 @@ int req_rerunjob( (!strncasecmp(preq->rq_extend, RERUNFORCE, strlen(RERUNFORCE)))) { std::string extend = RERUNFORCE; - batch_request *dup = duplicate_request(preq, -1); + batch_request *dup = new batch_request(*preq); get_batch_request_id(dup); - rc = issue_signal(&pjob, "SIGTERM", delay_and_send_sig_kill, extra, strdup(dup->rq_id)); + rc = issue_signal(&pjob, "SIGTERM", delay_and_send_sig_kill, extra, strdup(dup->rq_id.c_str())); if (rc == PBSE_NORELYMOM) { dup->rq_reply.brp_code = PBSE_NORELYMOM; pjob_mutex.unlock(); post_rerun(dup); + pjob = svr_find_job(preq->rq_ind.rq_signal.rq_jid, FALSE); if (pjob == NULL) + { + delete dup; return(PBSE_NONE); + } + pjob_mutex.set_lock_state(true); rc = PBSE_NONE; } + + delete dup; } else { - rc = issue_signal(&pjob, "SIGTERM", delay_and_send_sig_kill, extra, strdup(preq->rq_id)); + rc = issue_signal(&pjob, "SIGTERM", delay_and_send_sig_kill, extra, strdup(preq->rq_id.c_str())); if (rc != PBSE_NONE) { /* cant send to MOM */ diff --git a/src/server/req_runjob.c b/src/server/req_runjob.c index b7d985f834..17d2d4188d 100644 --- a/src/server/req_runjob.c +++ b/src/server/req_runjob.c @@ -140,12 +140,12 @@ extern int LOGLEVEL; /* Public Functions in this file */ -int svr_startjob(job *, struct batch_request **, char *, char *); +int svr_startjob(job *, batch_request *, char *, char *); /* Private Functions local to this file */ -int svr_stagein(job **, struct batch_request **, int, int); -int svr_strtjob2(job **, struct batch_request *); +int svr_stagein(job **, batch_request *, int, int); +int svr_strtjob2(job **, batch_request *); job *chk_job_torun(struct batch_request *, int); int assign_hosts(job *, char *, int, char *, char *); @@ -267,10 +267,9 @@ int check_and_run_job_work( /* NOTE: nodes assigned to job in svr_startjob() */ - rc = svr_startjob(pjob, &preq, failhost, emsg); + rc = svr_startjob(pjob, preq, failhost, emsg); - if ((rc != 0) && - (preq != NULL)) + if (rc != 0) { free_nodes(pjob); @@ -285,10 +284,6 @@ int check_and_run_job_work( } } - if ((rc == PBSE_NONE) && - (preq != NULL)) - free_br(preq); - return(rc); } // END check_and_run_job_work() @@ -303,6 +298,8 @@ void *check_and_run_job( check_and_run_job_work(preq); + delete preq; + return(NULL); } /* END check_and_run_job() */ @@ -370,8 +367,8 @@ int req_runjob( if (preq->rq_type == PBS_BATCH_AsyrunJob) { reply_ack(preq); - preq->rq_noreply = TRUE; - enqueue_threadpool_request(check_and_run_job, preq, async_pool); + preq->rq_noreply = true; + enqueue_threadpool_request(check_and_run_job, new batch_request(*preq), async_pool); } else { @@ -421,7 +418,6 @@ void post_checkpointsend( { int code; job *pjob; - bool preq_free_done = FALSE; pbs_attribute *pwait; char log_buf[LOCAL_LOG_BUF_SIZE]; @@ -490,18 +486,13 @@ void post_checkpointsend( job_save(pjob, SAVEJOB_FULL, 0); /* continue to start job running */ - svr_strtjob2(&pjob, preq); - preq_free_done = TRUE; } if (pjob == NULL) job_mutex.set_unlock_on_exit(false); } /* END if (pjob != NULL) */ - if (!preq_free_done) - free_br(preq); /* close connection and release request */ - return; } /* END post_checkpointsend() */ @@ -514,26 +505,23 @@ void post_checkpointsend( int svr_send_checkpoint( - job **pjob_ptr, /* I */ - struct batch_request **preq, /* I */ - int state, /* I */ - int substate) /* I */ + job **pjob_ptr, /* I */ + batch_request *preq, /* I */ + int state, /* I */ + int substate) /* I */ { - - struct batch_request *momreq = 0; - int rc; - char jobid[PBS_MAXSVRJOBID + 1]; - job *pjob = *pjob_ptr; + batch_request *momreq = 0; + int rc; + char jobid[PBS_MAXSVRJOBID + 1]; + job *pjob = *pjob_ptr; momreq = cpy_checkpoint(momreq, pjob, JOB_ATR_checkpoint_name, CKPT_DIR_IN); if (momreq == NULL) { /* no files to send, go directly to sending job to mom */ - rc = svr_strtjob2(&pjob, *preq); - - *preq = NULL; + rc = svr_strtjob2(&pjob, preq); return(rc); } @@ -563,25 +551,16 @@ int svr_send_checkpoint( * checkpoint copy started ok - reply to client as copy may * take too long to wait. */ - if (*preq != NULL) - { - reply_ack(*preq); - *preq = NULL; - } - } - else - { - free_br(momreq); + reply_ack(preq); } + + delete momreq; return(rc); } /* END svr_send_checkpoint() */ - - - /* * req_stagein - service the Stage In Files for a Job Request * @@ -622,9 +601,9 @@ int req_stagein( if ((rc = svr_stagein( &pjob, - &preq, - JOB_STATE_QUEUED, - JOB_SUBSTATE_STAGEIN))) + preq, + JOB_STATE_QUEUED, + JOB_SUBSTATE_STAGEIN))) { free_nodes(pjob); @@ -713,19 +692,14 @@ void post_stagein( /* need to copy checkpoint file to mom before running */ svr_send_checkpoint( &pjob, - &preq, + preq, JOB_STATE_RUNNING, JOB_SUBSTATE_CHKPTGO); } else { /* continue to start job running */ - svr_strtjob2(&pjob, preq); - - /* svr_strjob2 would call finish_sendmom which would free preq - in its reply_send_svr, set preq now to NULL to avoid double free */ - preq = NULL; } } else @@ -740,9 +714,6 @@ void post_stagein( job_mutex.set_unlock_on_exit(false); } /* END if (pjob != NULL) */ - if (preq) - free_br(preq); /* close connection and release request */ - return; } /* END post_stagein() */ @@ -756,14 +727,14 @@ void post_stagein( int svr_stagein( - job **pjob_ptr, /* I */ - struct batch_request **preq, /* I */ - int state, /* I */ - int substate) /* I */ + job **pjob_ptr, /* I */ + batch_request *preq, /* I */ + int state, /* I */ + int substate) /* I */ { job *pjob = *pjob_ptr; - struct batch_request *momreq = 0; + struct batch_request *momreq = NULL; int rc; char jobid[PBS_MAXSVRJOBID + 1]; @@ -772,9 +743,7 @@ int svr_stagein( if (momreq == NULL) { /* no files to stage, go directly to sending job to mom */ - rc = svr_strtjob2(&pjob, *preq); - - *preq = NULL; + rc = svr_strtjob2(&pjob, preq); return(rc); } @@ -807,30 +776,13 @@ int svr_stagein( * take too long to wait. */ - if (*preq != NULL) + if (preq != NULL) { - struct batch_request *request = *preq; - int free_preq = 0; - /* Under the following circumstances, reply_send_svr called eventually - * by reply_ack/reply_send will free preq under the following cirmcustances. - * There are 60+ occurrences of reply_send, so this is the easiest way. - */ - if (((request->rq_type != PBS_BATCH_AsyModifyJob) && - (request->rq_type != PBS_BATCH_AsyrunJob) && - (request->rq_type != PBS_BATCH_AsySignalJob)) || - (request->rq_noreply == TRUE)) - free_preq = 1; - - reply_ack(*preq); - - if (free_preq) - *preq = NULL; + reply_ack(preq); } } - else - { - free_br(momreq); - } + + delete momreq; return(rc); } /* END svr_stagein() */ @@ -987,10 +939,10 @@ int verify_moms_up( int svr_startjob( - job *pjob, /* I job to run (modified) */ - struct batch_request **preq, /* I Run Job batch request (optional) */ - char *FailHost, /* O (optional,minsize=1024) */ - char *EMsg) /* O (optional,minsize=1024) */ + job *pjob, /* I job to run (modified) */ + batch_request *preq, /* I Run Job batch request (optional) */ + char *FailHost, /* O (optional,minsize=1024) */ + char *EMsg) /* O (optional,minsize=1024) */ { int f; @@ -1113,9 +1065,7 @@ int svr_startjob( { /* No stage-in or already done, start job executing */ - rc = svr_strtjob2(&pjob, *preq); - - *preq = NULL; + rc = svr_strtjob2(&pjob, preq); } return(rc); @@ -1317,12 +1267,12 @@ int handle_heterogeneous_job_launch( lock_ji_mutex(cray_clone, __func__, NULL, LOGLEVEL); /* clone the batch requests to avoid double frees */ - external_preq = duplicate_request(preq); - cray_preq = duplicate_request(preq); + external_preq = new batch_request(*preq); + cray_preq = new batch_request(*preq); /* client doesn't need a response from these */ - external_preq->rq_noreply = TRUE; - cray_preq->rq_noreply = TRUE; + external_preq->rq_noreply = true; + cray_preq->rq_noreply = true; if ((rc = send_job_to_mom(&external_clone, external_preq, pjob)) == PBSE_NONE) { @@ -1334,8 +1284,9 @@ int handle_heterogeneous_job_launch( else both_running = TRUE; } - else - free_br(cray_preq); + + delete external_preq; + delete cray_preq; if (cray_clone != NULL) unlock_ji_mutex(cray_clone, __func__, NULL, LOGLEVEL); @@ -1375,8 +1326,8 @@ int handle_heterogeneous_job_launch( int svr_strtjob2( - job **pjob_ptr, /* I */ - struct batch_request *preq) /* I (modified - report status) */ + job **pjob_ptr, /* I */ + batch_request *preq) /* I (modified - report status) */ { job *pjob = *pjob_ptr; diff --git a/src/server/req_shutdown.c b/src/server/req_shutdown.c index 27c8f7e1bd..20395dd29b 100644 --- a/src/server/req_shutdown.c +++ b/src/server/req_shutdown.c @@ -110,8 +110,6 @@ #include "job_func.h" -int issue_signal(job **, const char *, void(*)(batch_request *), void *, char *); - /* Private Fuctions Local to this File */ /* Private Data Items */ diff --git a/src/server/req_signal.c b/src/server/req_signal.c index 755a4e54d2..3c8f888b49 100644 --- a/src/server/req_signal.c +++ b/src/server/req_signal.c @@ -136,9 +136,7 @@ int req_signaljob( job *pjob; int rc; char log_buf[LOCAL_LOG_BUF_SIZE]; - batch_request *dup_req = NULL; - /* preq free'd in error cases */ if ((pjob = chk_job_request(preq->rq_ind.rq_signal.rq_jid, preq)) == 0) { return(PBSE_NONE); @@ -202,49 +200,27 @@ int req_signaljob( /* send reply for asynchronous suspend */ if (preq->rq_type == PBS_BATCH_AsySignalJob) { - /* reply_ack will free preq. We need to copy it before we call reply_ack */ - batch_request *new_preq; - - new_preq = duplicate_request(preq, -1); - if (new_preq == NULL) - { - sprintf(log_buf, "failed to duplicate batch request"); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - return(PBSE_MEM_MALLOC); - } - - get_batch_request_id(new_preq); - - reply_ack(new_preq); - preq->rq_noreply = TRUE; + reply_ack(preq); + preq->rq_noreply = true; } /* pass the request on to MOM */ - if ((dup_req = duplicate_request(preq)) == NULL) + batch_request dup_req(*preq); + + rc = relay_to_mom(&pjob, &dup_req, NULL); + + if (pjob != NULL) + job_mutex.unlock(); + else + job_mutex.set_unlock_on_exit(false); + + if (rc != PBSE_NONE) { - req_reject(PBSE_SYSTEM, 0, preq, NULL, "can not allocate memory"); + req_reject(rc, 0, preq, NULL, NULL); /* unable to get to MOM */ } - /* The dup_req is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - else + else { - rc = relay_to_mom(&pjob, dup_req, NULL); - - if (pjob != NULL) - job_mutex.unlock(); - else - job_mutex.set_unlock_on_exit(false); - - if (rc != PBSE_NONE) - { - free_br(dup_req); - req_reject(rc, 0, preq, NULL, NULL); /* unable to get to MOM */ - } - else - { - post_signal_req(dup_req); - free_br(preq); - } + post_signal_req(&dup_req); } /* If successful we ack after mom replies to us, we pick up in post_signal_req() */ @@ -256,54 +232,51 @@ int req_signaljob( /* * issue_signal - send an internally generated signal to a running job + * + * @param pjob_ptr - a pointer to a potiner to the job + * @param signame - the name of the signal we're sending + * @param func - optional function pointer to call after sending the signal + * @param extra - optional extra info for the batch request + * @param extend - optional extend info for the batch request (basically more extra info) */ int issue_signal( job **pjob_ptr, - const char *signame, /* name of the signal to send */ + const char *signame, void (*func)(struct batch_request *), - void *extra, /* extra parameter to be stored in sig request */ - char *extend) /* Parameter to put in extended part of request */ + void *extra, + char *extend) { - int rc; - job *pjob = *pjob_ptr; - struct batch_request *newreq; - char jobid[PBS_MAXSVRJOBID + 1]; - - /* build up a Signal Job batch request */ - - if ((newreq = alloc_br(PBS_BATCH_SignalJob)) == NULL) - { - /* FAILURE */ - - return(PBSE_SYSTEM); - } + int rc; + job *pjob = *pjob_ptr; + batch_request newreq; + char jobid[PBS_MAXSVRJOBID + 1]; - newreq->rq_extra = extra; - newreq->rq_extend = extend; + newreq.rq_extra = extra; + newreq.rq_extend = extend; if (extend != NULL) { - newreq->rq_extsz = strlen(extend); + newreq.rq_extsz = strlen(extend); } strcpy(jobid, pjob->ji_qs.ji_jobid); - strcpy(newreq->rq_ind.rq_signal.rq_jid, pjob->ji_qs.ji_jobid); + strcpy(newreq.rq_ind.rq_signal.rq_jid, pjob->ji_qs.ji_jobid); - snprintf(newreq->rq_ind.rq_signal.rq_signame, sizeof(newreq->rq_ind.rq_signal.rq_signame), + snprintf(newreq.rq_ind.rq_signal.rq_signame, sizeof(newreq.rq_ind.rq_signal.rq_signame), "%s", signame); /* The newreq is freed in relay_to_mom (failure) * or in issue_Drequest (success) */ - rc = relay_to_mom(&pjob, newreq, NULL); + rc = relay_to_mom(&pjob, &newreq, NULL); if ((rc == PBSE_NONE) && (pjob != NULL)) { strcpy(jobid, pjob->ji_qs.ji_jobid); unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); - func(newreq); + func(&newreq); *pjob_ptr = svr_find_job(jobid, TRUE); } @@ -354,7 +327,9 @@ int issue_signal( svr_setjobstate(pjob, JOB_STATE_QUEUED, JOB_SUBSTATE_QUEUED, FALSE); unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); - func(newreq); + + if (func != NULL) + func(&newreq); rc = PBSE_NONE; @@ -366,8 +341,6 @@ int issue_signal( } else { - free_br(newreq); - if (pjob == NULL) *pjob_ptr = NULL; } diff --git a/src/server/req_stat.c b/src/server/req_stat.c index e3282dab0b..a160dafcec 100644 --- a/src/server/req_stat.c +++ b/src/server/req_stat.c @@ -731,16 +731,16 @@ int stat_to_mom( struct stat_cntl *cntl) /* M */ { - struct batch_request *newrq; - int rc = PBSE_NONE; - unsigned long addr; - char log_buf[LOCAL_LOG_BUF_SIZE+1]; - struct pbsnode *node; - int handle = -1; - unsigned long job_momaddr = -1; - unsigned short job_momport = -1; - char *job_momname = NULL; - job *pjob = NULL; + batch_request newrq(PBS_BATCH_StatusJob); + int rc = PBSE_NONE; + unsigned long addr; + char log_buf[LOCAL_LOG_BUF_SIZE+1]; + pbsnode *node; + int handle = -1; + unsigned long job_momaddr = -1; + unsigned short job_momport = -1; + char *job_momname = NULL; + job *pjob = NULL; if ((pjob = svr_find_job(job_id, FALSE)) == NULL) return(PBSE_JOBNOTFOUND); @@ -765,18 +765,12 @@ int stat_to_mom( if (job_momname == NULL) return PBSE_MEM_MALLOC; - if ((newrq = alloc_br(PBS_BATCH_StatusJob)) == NULL) - { - free(job_momname); - return PBSE_MEM_MALLOC; - } - if (cntl->sc_type == 1) - snprintf(newrq->rq_ind.rq_status.rq_id, sizeof(newrq->rq_ind.rq_status.rq_id), "%s", job_id); + snprintf(newrq.rq_ind.rq_status.rq_id, sizeof(newrq.rq_ind.rq_status.rq_id), "%s", job_id); else - newrq->rq_ind.rq_status.rq_id[0] = '\0'; /* get stat of all */ + newrq.rq_ind.rq_status.rq_id[0] = '\0'; /* get stat of all */ - CLEAR_HEAD(newrq->rq_ind.rq_status.rq_attr); + CLEAR_HEAD(newrq.rq_ind.rq_status.rq_attr); /* if MOM is down just return stale information */ addr = job_momaddr; @@ -785,7 +779,10 @@ int stat_to_mom( free(job_momname); if (node == NULL) + { return PBSE_UNKNODE; + } + if ((node->nd_state & INUSE_NOT_READY)||(node->nd_power_state != POWER_STATE_RUNNING)) { if (LOGLEVEL >= 6) @@ -798,7 +795,6 @@ int stat_to_mom( } node->unlock_node(__func__, "no rely mom", LOGLEVEL); - free_br(newrq); return PBSE_NORELYMOM; } @@ -809,9 +805,9 @@ int stat_to_mom( if (handle >= 0) { - if ((rc = issue_Drequest(handle, newrq, true)) == PBSE_NONE) + if ((rc = issue_Drequest(handle, &newrq, true)) == PBSE_NONE) { - stat_update(newrq, cntl); + stat_update(&newrq, cntl); } } else @@ -820,8 +816,6 @@ int stat_to_mom( if (rc == PBSE_SYSTEM) rc = PBSE_MEM_MALLOC; - free_br(newrq); - return(rc); } /* END stat_to_mom() */ @@ -835,8 +829,8 @@ int stat_to_mom( void stat_update( - struct batch_request *preq, - struct stat_cntl *cntl) + batch_request *preq, + struct stat_cntl *cntl) { job *pjob; diff --git a/src/server/req_track.c b/src/server/req_track.c index 8614b4b696..187b91f0c2 100644 --- a/src/server/req_track.c +++ b/src/server/req_track.c @@ -332,10 +332,9 @@ void issue_track( job *pjob) { - struct batch_request *preq; - char *pc; - char *sname; - char log_buf[LOCAL_LOG_BUF_SIZE]; + char *pc; + char *sname; + char log_buf[LOCAL_LOG_BUF_SIZE]; if ((pc = strchr(pjob->ji_qs.ji_jobid, '.')) == NULL) { @@ -359,17 +358,14 @@ void issue_track( return; } - preq = alloc_br(PBS_BATCH_TrackJob); + batch_request preq(PBS_BATCH_TrackJob); - if (preq == NULL) - return; - - preq->rq_ind.rq_track.rq_hopcount = pjob->ji_wattr[JOB_ATR_hopcount].at_val.at_long; + preq.rq_ind.rq_track.rq_hopcount = pjob->ji_wattr[JOB_ATR_hopcount].at_val.at_long; - strcpy(preq->rq_ind.rq_track.rq_jid, pjob->ji_qs.ji_jobid); - strcpy(preq->rq_ind.rq_track.rq_location, server_name); + strcpy(preq.rq_ind.rq_track.rq_jid, pjob->ji_qs.ji_jobid); + strcpy(preq.rq_ind.rq_track.rq_location, server_name); - preq->rq_ind.rq_track.rq_state[0] = pjob->ji_wattr[JOB_ATR_state].at_val.at_char; + preq.rq_ind.rq_track.rq_state[0] = pjob->ji_wattr[JOB_ATR_state].at_val.at_char; pc = pjob->ji_qs.ji_jobid; @@ -377,6 +373,4 @@ void issue_track( pc++; issue_to_svr(++pc, &preq, NULL); - if (preq != NULL) - free_br(preq); } // END issue_track() diff --git a/src/server/svr_process_request.h b/src/server/svr_process_request.h index 753ee3ca64..a235fea16b 100644 --- a/src/server/svr_process_request.h +++ b/src/server/svr_process_request.h @@ -12,12 +12,8 @@ int dispatch_request(int sfds, struct batch_request *request); static void svr_close_client(int sfds); -struct batch_request *alloc_br(int type); - static void close_quejob(int sfds); -void free_br(struct batch_request *preq); - static void freebr_manage(struct rq_manage *pmgr); static void freebr_cpyfile(struct rq_cpyfile *pcf); diff --git a/src/test/array_func/scaffolding.c b/src/test/array_func/scaffolding.c index 24bb20b9d7..9e667cffd1 100644 --- a/src/test/array_func/scaffolding.c +++ b/src/test/array_func/scaffolding.c @@ -379,3 +379,10 @@ int create_and_queue_array_subjob( { return(PBSE_NONE); } + +batch_request::batch_request(const batch_request &other) + { + } + +void batch_request::update_object_id(int job_index) {} + diff --git a/src/test/batch_request/scaffolding.c b/src/test/batch_request/scaffolding.c index c4730e4b73..e909f6eacd 100644 --- a/src/test/batch_request/scaffolding.c +++ b/src/test/batch_request/scaffolding.c @@ -9,3 +9,10 @@ int LOGLEVEL = 10; void log_err(int errnum, const char *routine, const char *text) {} + +void reply_free( + + struct batch_reply *prep) + + { + } diff --git a/src/test/batch_request/test_uut.c b/src/test/batch_request/test_uut.c index fe64a25829..f121aa76d7 100644 --- a/src/test/batch_request/test_uut.c +++ b/src/test/batch_request/test_uut.c @@ -7,10 +7,10 @@ START_TEST(test_one) { - batch_request *pBr1 = (batch_request *)calloc(1,sizeof(batch_request)); - batch_request *pBr2 = (batch_request *)calloc(1,sizeof(batch_request)); - batch_request *pBr3 = (batch_request *)calloc(1,sizeof(batch_request)); - batch_request *pBr4 = (batch_request *)calloc(1,sizeof(batch_request)); + batch_request *pBr1 = new (batch_request); + batch_request *pBr2 = new (batch_request); + batch_request *pBr3 = new (batch_request); + batch_request *pBr4 = new (batch_request); int rc = get_batch_request_id(pBr1); fail_unless(rc == PBSE_NONE); @@ -41,13 +41,49 @@ START_TEST(test_one) END_TEST +START_TEST(test_constructor) + { + batch_request *preq = new batch_request(PBS_BATCH_QueueJob); + fail_unless(preq->rq_type == PBS_BATCH_QueueJob); + fail_unless(preq->rq_conn == -1); + fail_unless(preq->rq_orgconn == -1); + fail_unless(preq->rq_reply.brp_choice == BATCH_REPLY_CHOICE_NULL); + fail_unless(preq->rq_noreply == FALSE); + fail_unless(preq->rq_time > 0); + delete preq; + } +END_TEST -START_TEST(test_two) + +START_TEST(test_br_copy_constructor) { + batch_request preq; + batch_request *dup; + + preq.rq_perm = 1; + strcpy(preq.rq_user, "dbeer"); + strcpy(preq.rq_host, "napali"); + preq.rq_extend = strdup("tom"); + preq.rq_type = PBS_BATCH_RunJob; + preq.rq_ind.rq_run.rq_destin = strdup("napali"); + + dup = new batch_request(preq); + fail_unless(dup != NULL); + fail_unless(!strcmp(dup->rq_extend, "tom")); + fail_unless(!strcmp(dup->rq_user, "dbeer")); + fail_unless(!strcmp(dup->rq_host, "napali")); + fail_unless(!strcmp(dup->rq_extend, "tom")); + fail_unless(!strcmp(dup->rq_ind.rq_run.rq_destin, "napali")); + + preq.rq_type = PBS_BATCH_Rerun; + const char *rerun_jobid = "4.roshar"; + strcpy(preq.rq_ind.rq_rerun, rerun_jobid); + batch_request *rerun_dep = new batch_request(preq); + fail_unless(!strcmp(rerun_dep->rq_ind.rq_rerun, rerun_jobid)); } -END_TEST +END_TEST @@ -58,10 +94,11 @@ Suite *batch_request_suite(void) TCase *tc_core = tcase_create("test_one"); tcase_add_test(tc_core, test_one); + tcase_add_test(tc_core, test_br_copy_constructor); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_constructor"); + tcase_add_test(tc_core, test_constructor); suite_add_tcase(s, tc_core); return(s); diff --git a/src/test/catch_child/scaffolding.c b/src/test/catch_child/scaffolding.c index 3b4d76e644..4daa1ccaed 100644 --- a/src/test/catch_child/scaffolding.c +++ b/src/test/catch_child/scaffolding.c @@ -1418,3 +1418,7 @@ int send_back_std_and_staged_files(job *pjob, int exit_value) task::~task() {} + +batch_request::batch_request(int type) : rq_type(type) + { + } diff --git a/src/test/display_alps_status/scaffolding.c b/src/test/display_alps_status/scaffolding.c index f784959185..8852ebfab8 100644 --- a/src/test/display_alps_status/scaffolding.c +++ b/src/test/display_alps_status/scaffolding.c @@ -43,3 +43,6 @@ int pbsnode::unlock_node(const char *id, const char *msg, int level) return(0); } +batch_request::batch_request() {} +batch_request::~batch_request() {} + diff --git a/src/test/issue_request/scaffolding.c b/src/test/issue_request/scaffolding.c index 1d79195ec6..311afbb7bc 100644 --- a/src/test/issue_request/scaffolding.c +++ b/src/test/issue_request/scaffolding.c @@ -274,40 +274,24 @@ void DIS_tcp_cleanup(struct tcp_chan *chan) { } -/* - * alloc_br - allocate and clear a batch_request structure - */ - -struct batch_request *alloc_br( +batch_request::batch_request( int type) { + this->rq_type = type; - struct batch_request *req = NULL; - - if ((req = (struct batch_request *)calloc(1, sizeof(struct batch_request))) == NULL) - { - fprintf(stderr, "failed to allocate batch request. alloc_br()\n"); - } - else - { - - req->rq_type = type; - - req->rq_conn = -1; /* indicate not connected */ - req->rq_orgconn = -1; /* indicate not connected */ - req->rq_time = time(NULL); - req->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; - req->rq_noreply = FALSE; /* indicate reply is needed */ - } + this->rq_conn = -1; /* indicate not connected */ + this->rq_orgconn = -1; /* indicate not connected */ + this->rq_time = time(NULL); + this->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; + this->rq_noreply = FALSE; /* indicate reply is needed */ - return(req); - } /* END alloc_br() */ + } batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -399,7 +383,7 @@ int copy_attribute_list( batch_request *duplicate_request(batch_request *preq, int job_index) { - batch_request *preq_tmp = alloc_br(preq->rq_type); + batch_request *preq_tmp = new batch_request(preq->rq_type); char *ptr1; char *ptr2; char newjobname[PBS_MAXSVRJOBID+1]; @@ -544,3 +528,8 @@ void set_reply_type(struct batch_reply *preply, int type) { preply->brp_choice = type; } + +batch_request::batch_request() {} +batch_request::batch_request(const batch_request &other) {} +batch_request::~batch_request() {} + diff --git a/src/test/issue_request/test_uut.c b/src/test/issue_request/test_uut.c index 04f878a172..fabd80263e 100644 --- a/src/test/issue_request/test_uut.c +++ b/src/test/issue_request/test_uut.c @@ -22,7 +22,7 @@ int issue_Drequest(int conn, batch_request *request, bool close_handle); START_TEST(queue_a_retry_task_test) { - batch_request *preq = (batch_request *)calloc(1, sizeof(batch_request)); + batch_request *preq = new batch_request(); preq->rq_id = strdup("tom"); queue_a_retry_task(preq, NULL); @@ -40,7 +40,6 @@ START_TEST(test_one) job *pTestJob; struct batch_request request; - memset(&request,0,sizeof(request)); pTestJob = &testJob; local_connect = true; @@ -59,7 +58,7 @@ START_TEST(test_send_request_to_remote_server) int rc; batch_request *preq; - preq = alloc_br(PBS_BATCH_TrackJob); + preq = new batch_request(PBS_BATCH_TrackJob); fail_unless(preq != NULL); preq->rq_ind.rq_track.rq_hopcount = 1; strcpy(preq->rq_ind.rq_track.rq_jid, "1234"); @@ -73,20 +72,20 @@ START_TEST(test_send_request_to_remote_server) free(preq); - preq = alloc_br(PBS_BATCH_DeleteJob); + preq = new batch_request(PBS_BATCH_DeleteJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_track.rq_jid, "1234"); rc = send_request_to_remote_server(10, preq, true); fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_CheckpointJob); + preq = new batch_request(PBS_BATCH_CheckpointJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_hold.rq_orig.rq_objname, "some_object"); rc = send_request_to_remote_server(10, preq, true); free(preq); - preq = alloc_br(PBS_BATCH_GpuCtrl); + preq = new batch_request(PBS_BATCH_GpuCtrl); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_gpuctrl.rq_momnode, "nodea"); strcpy(preq->rq_ind.rq_gpuctrl.rq_gpuid, "0"); @@ -98,7 +97,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_MessJob); + preq = new batch_request(PBS_BATCH_MessJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_message.rq_jid, "1234"); preq->rq_ind.rq_message.rq_file = 1; @@ -109,7 +108,7 @@ START_TEST(test_send_request_to_remote_server) free(preq); - preq = alloc_br(PBS_BATCH_Rerun); + preq = new batch_request(PBS_BATCH_Rerun); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_rerun, "1234"); @@ -117,7 +116,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_RegistDep); + preq = new batch_request(PBS_BATCH_RegistDep); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_rerun, "1234"); @@ -125,7 +124,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_AsySignalJob); + preq = new batch_request(PBS_BATCH_AsySignalJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_signal.rq_jid, "1234"); strcpy(preq->rq_ind.rq_signal.rq_signame, "SIGKILL"); @@ -135,7 +134,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_SignalJob); + preq = new batch_request(PBS_BATCH_SignalJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_signal.rq_jid, "1234"); strcpy(preq->rq_ind.rq_signal.rq_signame, "SIGKILL"); @@ -145,7 +144,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_StatusJob); + preq = new batch_request(PBS_BATCH_StatusJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -153,7 +152,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_ReturnFiles); + preq = new batch_request(PBS_BATCH_ReturnFiles); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -161,7 +160,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_CopyFiles); + preq = new batch_request(PBS_BATCH_CopyFiles); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -169,7 +168,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_DelFiles); + preq = new batch_request(PBS_BATCH_DelFiles); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -177,7 +176,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_DeleteReservation); + preq = new batch_request(PBS_BATCH_DeleteReservation); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -185,7 +184,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_GAP038); + preq = new batch_request(PBS_BATCH_GAP038); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -203,7 +202,7 @@ START_TEST(test_handle_local_request) int rc; struct batch_request *preq; - preq = alloc_br(PBS_BATCH_StatusJob); + preq = new batch_request(PBS_BATCH_StatusJob); fail_unless(preq != NULL); rc = handle_local_request(10, preq); @@ -218,7 +217,7 @@ START_TEST(test_issue_Drequest) int rc; struct batch_request *preq; - preq = alloc_br(PBS_BATCH_StatusJob); + preq = new batch_request(PBS_BATCH_StatusJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -236,37 +235,35 @@ START_TEST(test_issue_to_svr) char server_name[PBS_MAXHOSTNAME]; struct batch_request *preq; - preq = alloc_br(PBS_BATCH_StatusJob); + preq = new batch_request(PBS_BATCH_StatusJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); strcpy(server_name, "hosta"); return_addr = true; local_connect = true; - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_NONE); - // local connect should cause preq to be set to NULL - fail_unless(preq == NULL); return_addr = false; - preq = alloc_br(PBS_BATCH_StatusJob); - rc = issue_to_svr(server_name, &preq, NULL); + preq = new batch_request(PBS_BATCH_StatusJob); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_NONE); return_addr = true; local_connect = false; net_rc_retry = false; - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_NONE); net_rc_retry = true; - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_NONE); local_connect = false; net_rc_retry = false; connect_error = true; - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_INTERNAL); } @@ -282,7 +279,7 @@ START_TEST(test_reissue_to_svr); reissue_to_svr(pwt); /* Schedule a task. Success */ - preq = alloc_br(PBS_BATCH_TrackJob); + preq = new batch_request(PBS_BATCH_TrackJob); fail_unless(preq != NULL); preq->rq_ind.rq_track.rq_hopcount = 1; strcpy(preq->rq_ind.rq_track.rq_jid, "1234"); @@ -293,7 +290,7 @@ START_TEST(test_reissue_to_svr); preq->rq_perm = ATR_DFLAG_MGRD | ATR_DFLAG_MGWR | ATR_DFLAG_SvWR; get_batch_request_id(preq); - pwt = set_task(WORK_Timed, (time(NULL) + PBS_NET_RETRY_TIME), reissue_to_svr, preq->rq_id, TRUE); + pwt = set_task(WORK_Timed, (time(NULL) + PBS_NET_RETRY_TIME), reissue_to_svr, strdup(preq->rq_id.c_str()), TRUE); fail_unless(pwt != NULL); reissue_to_svr(pwt); diff --git a/src/test/job_func/scaffolding.c b/src/test/job_func/scaffolding.c index 51a4626a65..f4637d67d0 100644 --- a/src/test/job_func/scaffolding.c +++ b/src/test/job_func/scaffolding.c @@ -531,12 +531,11 @@ ssize_t read_ac_socket(int fd, void *buf, ssize_t count) return(0); } -struct batch_request *alloc_br(int type) +batch_request::batch_request( + + int type) : rq_type(type) + { - struct batch_request *request = NULL; - request = (batch_request *)calloc(1, sizeof(struct batch_request)); - request->rq_type = type; - return request; } void free_str(struct pbs_attribute *attr) {} @@ -715,3 +714,6 @@ bool job_array::is_deleted() const { return(this->being_deleted); } + +batch_request::~batch_request() {} + diff --git a/src/test/job_func/test_uut.c b/src/test/job_func/test_uut.c index 537d6067db..cb78ea1dde 100644 --- a/src/test/job_func/test_uut.c +++ b/src/test/job_func/test_uut.c @@ -307,7 +307,7 @@ START_TEST(cpy_checkpoint_test) test_job, JOB_ATR_checkpoint_name, CKPT_DIR_IN); - struct batch_request *initial = alloc_br(/*PBS_BATCH_CheckpointJob*/0); + struct batch_request *initial = new batch_request(0); fail_unless(result == NULL, "NULL batch_request input fail"); result = cpy_checkpoint(initial, diff --git a/src/test/job_recov/scaffolding.c b/src/test/job_recov/scaffolding.c index 27eadb63a7..cd4a8b0f03 100644 --- a/src/test/job_recov/scaffolding.c +++ b/src/test/job_recov/scaffolding.c @@ -283,12 +283,16 @@ int set_tokens(pbs_attribute *attr, pbs_attribute *newAttr, enum batch_op op){re int comp_ll(struct pbs_attribute *attr, struct pbs_attribute *with) {return 0;} int decode_b(pbs_attribute *patr, const char *name, const char *rn, const char *val, int) {return 0;} int nextjobnum_chk(pbs_attribute *pattr, void *pobject, int actmode) {return 0;} -struct batch_request *alloc_br(int type) {return NULL;} int svr_chk_owner(struct batch_request *preq, job *pjob) {return 0;} int comp_checkpoint(pbs_attribute *attr, pbs_attribute *with) {return 0;} batch_request *get_remove_batch_request(char *br_id) {return NULL;} long calc_job_cost(job *pjob) {return(0);} -int issue_to_svr(const char *servern, struct batch_request **preq, void (*replyfunc)(struct work_task *)) {return 0;} + +int issue_to_svr(const char *servern, batch_request *preq, void (*replyfunc)(struct work_task *)) + { + return 0; + } + int que_to_local_svr(struct batch_request *preq) {return 0;} int job_set_wait(pbs_attribute *pattr, void *pjob, int mode) {return 0;} int get_batch_request_id(batch_request *preq) {return 0;} @@ -534,3 +538,17 @@ int check_default_gpu_mode_str(pbs_attribute *pattr, void *pobj, int mode) { return(PBSE_NONE); } + +batch_request::~batch_request() {} + +batch_request::batch_request(int type) : rq_type(type) + { + } + +batch_request *get_remove_batch_request( + + const char *br_id) + + { + return(NULL); + } diff --git a/src/test/mom_req_quejob/scaffolding.c b/src/test/mom_req_quejob/scaffolding.c index d59f57ca9b..583f5a8813 100644 --- a/src/test/mom_req_quejob/scaffolding.c +++ b/src/test/mom_req_quejob/scaffolding.c @@ -229,3 +229,6 @@ void free_pwnam(struct passwd *pwdp, char *buf) void send_update_soon() {} + +batch_request::batch_request() {} +batch_request::~batch_request() {} diff --git a/src/test/mom_req_quejob/test_uut.c b/src/test/mom_req_quejob/test_uut.c index e7e04e382b..22866c9b7b 100644 --- a/src/test/mom_req_quejob/test_uut.c +++ b/src/test/mom_req_quejob/test_uut.c @@ -15,7 +15,6 @@ START_TEST(test_mom_req_quejob) { batch_request preq; - memset(&preq, 0, sizeof(preq)); preq.rq_fromsvr = TRUE; strcpy(preq.rq_ind.rq_queuejob.rq_jid, "1.napali"); CLEAR_HEAD(preq.rq_ind.rq_queuejob.rq_attr); diff --git a/src/test/node_manager/scaffolding.c b/src/test/node_manager/scaffolding.c index 508eb765d7..c69aded719 100644 --- a/src/test/node_manager/scaffolding.c +++ b/src/test/node_manager/scaffolding.c @@ -1092,3 +1092,7 @@ pbsnode *authorized_hosts::get_authorized_node(unsigned long addr, unsigned shor authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; + +batch_request::batch_request(int type) {} +batch_request::~batch_request() {} + diff --git a/src/test/node_manager/test_uut.c b/src/test/node_manager/test_uut.c index 71dc19c128..476dd67440 100644 --- a/src/test/node_manager/test_uut.c +++ b/src/test/node_manager/test_uut.c @@ -226,11 +226,7 @@ START_TEST(test_kill_job_on_mom) rc = kill_job_on_mom(job_id, &pnode); fail_unless(rc == PBSE_NONE); - alloc_br_success = false; - rc = kill_job_on_mom(job_id, &pnode); - fail_unless(rc == -1); - - alloc_br_success = true;; + alloc_br_success = true; conn_success = false; rc = kill_job_on_mom(job_id, &pnode); fail_unless(rc == -1); diff --git a/src/test/process_alps_status/scaffolding.c b/src/test/process_alps_status/scaffolding.c index 55cea6d1d9..dd8ab2c577 100644 --- a/src/test/process_alps_status/scaffolding.c +++ b/src/test/process_alps_status/scaffolding.c @@ -1914,3 +1914,7 @@ void Machine::setMemory(long long mem) } #endif +batch_request::batch_request(int type) {} +batch_request::batch_request() {} +batch_request::~batch_request() {} + diff --git a/src/test/process_request/scaffolding.c b/src/test/process_request/scaffolding.c index 8750c4bf79..66618f8900 100644 --- a/src/test/process_request/scaffolding.c +++ b/src/test/process_request/scaffolding.c @@ -553,12 +553,16 @@ int req_job_cleanup_done(batch_request *preq) } int req_quejob2(batch_request *preq) - { - return(0); - } + { + return(0); + } int req_commit2(batch_request *preq) - { - return(0); - } + { + return(0); + } + +batch_request::batch_request() {} +batch_request::~batch_request() {} + diff --git a/src/test/process_request/test_uut.c b/src/test/process_request/test_uut.c index d6c6331ddf..704ef6de61 100644 --- a/src/test/process_request/test_uut.c +++ b/src/test/process_request/test_uut.c @@ -12,8 +12,7 @@ extern struct connection svr_conn[]; int process_request(struct tcp_chan *chan); bool request_passes_acl_check(batch_request *request, unsigned long conn_addr); -batch_request *alloc_br(int type); -batch_request *read_request_from_socket(tcp_chan *chan); +int read_request_from_socket(tcp_chan *chan, batch_request &preq); extern bool check_acl; extern bool find_node; @@ -46,43 +45,43 @@ void set_connection_type( START_TEST(test_read_request_from_socket) { - tcp_chan chan; + tcp_chan chan; + batch_request preq; memset(&chan, 0, sizeof(chan)); chan.sock = -1; - fail_unless(read_request_from_socket(&chan) == NULL, "invalid socket (-1) should return NULL"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "invalid socket (-1) should return NULL"); chan.sock = PBS_NET_MAX_CONNECTIONS; - fail_unless(read_request_from_socket(&chan) == NULL, "invalid socket (PBS_NET_MAX_CONNECTIONS) should return NULL"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "invalid socket (PBS_NET_MAX_CONNECTIONS) should return NULL"); chan.sock = 1; initialize_svr_conn(chan.sock); set_connection_type(chan.sock, Idle); - fail_unless(read_request_from_socket(&chan) == NULL, "Idle connection type should fail"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "Idle connection type should fail"); strcpy(server_name, "napali"); set_connection_type(chan.sock, ToServerDIS); find_node = false; check_acl = true; fail_check = false; - fail_unless(read_request_from_socket(&chan) == NULL, "Node not found should fail"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "Node not found should fail"); fail_check = true; find_node = true; check_acl = false; - fail_unless(read_request_from_socket(&chan) != NULL, "should pass"); + fail_unless(read_request_from_socket(&chan, preq) == PBSE_NONE, "should pass"); fail_get_connecthost = true; - fail_unless(read_request_from_socket(&chan) == NULL, "fail if can't get connect host"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "fail if can't get connect host"); dis_req_read_rc = PBSE_SYSTEM; - batch_request *preq; - fail_unless((preq = read_request_from_socket(&chan)) != NULL, "should get bad request"); - fail_unless(preq->rq_type == PBS_BATCH_Disconnect); + fail_unless((read_request_from_socket(&chan, preq)) == PBSE_SYSTEM, "should get bad request: %d"); + fail_unless(preq.rq_type == PBS_BATCH_Disconnect); dis_req_read_rc = 5; - fail_unless((preq = read_request_from_socket(&chan)) != NULL, "should get bad request"); - fail_unless(preq->rq_failcode == dis_req_read_rc); + fail_unless((read_request_from_socket(&chan, preq)) == dis_req_read_rc, "should get bad request"); + fail_unless(preq.rq_failcode == dis_req_read_rc); } END_TEST @@ -116,22 +115,6 @@ START_TEST(test_request_passes_acl_check) } END_TEST -START_TEST(test_alloc_br) - { - batch_request *preq = alloc_br(PBS_BATCH_QueueJob); - - fail_unless(preq->rq_type == PBS_BATCH_QueueJob); - fail_unless(preq->rq_conn == -1); - fail_unless(preq->rq_orgconn == -1); - fail_unless(preq->rq_reply.brp_choice == BATCH_REPLY_CHOICE_NULL); - fail_unless(preq->rq_noreply == FALSE); - fail_unless(preq->rq_time > 0); - - free_br(preq); - fail_unless(free_attrlist_called > 0); - } -END_TEST - START_TEST(test_process_request_bad_host_err) { struct tcp_chan chan; @@ -161,7 +144,7 @@ Suite *process_request_suite(void) tc_core = tcase_create("test_request_passes_acl_check"); tcase_add_test(tc_core, test_request_passes_acl_check); - tcase_add_test(tc_core, test_alloc_br); + suite_add_tcase(s, tc_core); tc_core = tcase_create("test_process_request_bad_host_err"); tcase_add_test(tc_core, test_process_request_bad_host_err); diff --git a/src/test/req_delete/scaffolding.c b/src/test/req_delete/scaffolding.c index b4ef218c19..db6d2ab44d 100644 --- a/src/test/req_delete/scaffolding.c +++ b/src/test/req_delete/scaffolding.c @@ -41,7 +41,7 @@ int depend_term_called; batch_request *alloc_br(int type) { if (alloc_work) - return((batch_request *)calloc(1, sizeof(batch_request))); + return(new batch_request()); else return(NULL); } @@ -66,13 +66,12 @@ int job_save(job *pjob, int updatetype, int mom_port) int svr_job_purge(job *pjob, int leaveSpoolFiles) { pjob->ji_qs.ji_state = JOB_STATE_COMPLETE; + pthread_mutex_unlock(pjob->ji_mutex); return(0); } void chk_job_req_permissions(job **pjob_ptr, struct batch_request *preq) { - fprintf(stderr, "The call to chk_job_req_permissions needs to be mocked!!\n"); - exit(1); } void svr_mailowner(job *pjob, int mailpoint, int force, const char *text) @@ -113,7 +112,6 @@ void free_nodes(job *pjob, const char *spec) void free_br(struct batch_request *preq) { - br_freed = true; } struct work_task *set_task(enum work_type type, long event_id, void (*func)(struct work_task *), void *parm, int get_lock) @@ -163,7 +161,7 @@ char *pbse_to_txt(int err) batch_request *cpy_stage(batch_request *preq, job *pjob, enum job_atr ati, int direction) { - return((batch_request *)calloc(1, sizeof(batch_request))); + return(new batch_request()); } int svr_setjobstate(job *pjob, int newstate, int newsubstate, int has_queue_mute) @@ -228,7 +226,7 @@ int get_svr_attr_b(int index, bool *b) batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -579,6 +577,8 @@ int depend_on_term( job::job() : ji_has_delete_nanny(false) { memset(this->ji_wattr, 0, sizeof(this->ji_wattr)); + this->ji_mutex = (pthread_mutex_t *)calloc(1, sizeof(pthread_mutex_t)); + pthread_mutex_init(this->ji_mutex, NULL); } job::~job() {} @@ -595,4 +595,19 @@ void job_array::update_array_values( void job_array::mark_deleted() {} +batch_request::batch_request(const batch_request &other) + { + if (other.rq_extend == NULL) + this->rq_extend = NULL; + else + this->rq_extend = strdup(other.rq_extend); + } + +batch_request::~batch_request() + + { + br_freed = true; + } +batch_request::batch_request() {} +batch_request::batch_request(int type) : rq_type(type) {} diff --git a/src/test/req_delete/test_uut.c b/src/test/req_delete/test_uut.c index 6c2f3c5a46..8842037da0 100644 --- a/src/test/req_delete/test_uut.c +++ b/src/test/req_delete/test_uut.c @@ -77,7 +77,7 @@ int set_pbs_server_name() START_TEST(test_handle_single_delete) { - batch_request *preq = (batch_request *)calloc(1,sizeof(batch_request)); + batch_request *preq = new batch_request(); strcpy(preq->rq_ind.rq_delete.rq_objname, "2.napali"); fail_unless(handle_single_delete(preq, preq, NULL) == PBSE_NONE); fail_unless(preq->rq_noreply == FALSE); @@ -91,44 +91,12 @@ END_TEST START_TEST(test_handle_delete_all) { batch_request preq; - memset(&preq, 0, sizeof(preq)); fail_unless(handle_delete_all(&preq, &preq, NULL) == PBSE_NONE); fail_unless(preq.rq_noreply == TRUE); } END_TEST -START_TEST(test_duplicate_request) - { - batch_request *preq = (batch_request *)calloc(1, sizeof(batch_request)); - batch_request *dup; - alloc_work = 0; - fail_unless(duplicate_request(preq) == NULL); - - alloc_work = 1; - preq->rq_perm = 1; - strcpy(preq->rq_user, "dbeer"); - strcpy(preq->rq_host, "napali"); - preq->rq_extend = strdup("tom"); - preq->rq_type = PBS_BATCH_RunJob; - preq->rq_ind.rq_run.rq_destin = strdup("napali"); - - dup = duplicate_request(preq); - fail_unless(dup != NULL); - fail_unless(!strcmp(dup->rq_extend, "tom")); - fail_unless(!strcmp(dup->rq_user, "dbeer")); - fail_unless(!strcmp(dup->rq_host, "napali")); - fail_unless(!strcmp(dup->rq_extend, "tom")); - fail_unless(!strcmp(dup->rq_ind.rq_run.rq_destin, "napali")); - - preq->rq_type = PBS_BATCH_Rerun; - const char *rerun_jobid = "4.roshar"; - strcpy(preq->rq_ind.rq_rerun, rerun_jobid); - batch_request *rerun_dep = duplicate_request(preq, -1); - fail_unless(!strcmp(rerun_dep->rq_ind.rq_rerun, rerun_jobid)); - } -END_TEST - START_TEST(test_post_delete_mom2) { struct work_task *ptask; @@ -154,7 +122,7 @@ START_TEST(test_forced_jobpurge) batch_request *preq; pjob = new job(); - preq = (batch_request *)calloc(1, sizeof(batch_request)); + preq = new batch_request(); strcpy(pjob->ji_qs.ji_jobid, "1.napali"); memset(pjob->ji_arraystructid, 0, sizeof(pjob->ji_arraystructid)); @@ -179,10 +147,8 @@ START_TEST(test_delete_all_work) batch_request *preq; pjob = new job(); - pjob->ji_mutex = (pthread_mutex_t *)calloc(1, sizeof(pthread_mutex_t)); - pthread_mutex_init(pjob->ji_mutex,NULL); - preq = (batch_request *)calloc(1, sizeof(batch_request)); + preq = new batch_request(); preq->rq_extend = strdup(delpurgestr); nanny = 0; @@ -190,7 +156,8 @@ START_TEST(test_delete_all_work) // no lock should remain on job after delete_all_work() // so test by making sure we can set lock - fail_unless(delete_all_work((void *) preq) == NULL && pthread_mutex_trylock(pjob->ji_mutex) == 0); + fail_unless(delete_all_work((void *) preq) == NULL); + fail_unless(pthread_mutex_trylock(pjob->ji_mutex) == 0); nanny = 1; } @@ -200,37 +167,22 @@ START_TEST(test_post_job_delete_nanny) { batch_request *preq_sig; - br_freed = FALSE; - post_job_delete_nanny(NULL); - fail_unless(br_freed == FALSE); - - preq_sig = (batch_request *)calloc(1, sizeof(batch_request)); - br_freed = FALSE; + preq_sig = new batch_request(); nanny = 0; post_job_delete_nanny(preq_sig); - fail_unless(br_freed == TRUE); - preq_sig = (batch_request *)calloc(1, sizeof(batch_request)); - br_freed = FALSE; nanny = 1; strcpy(preq_sig->rq_ind.rq_signal.rq_jid, "2.napali"); post_job_delete_nanny(preq_sig); - fail_unless(br_freed == TRUE); - preq_sig = (batch_request *)calloc(1, sizeof(batch_request)); - br_freed = FALSE; nanny = 1; strcpy(preq_sig->rq_ind.rq_signal.rq_jid, "1.napali"); post_job_delete_nanny(preq_sig); - fail_unless(br_freed == TRUE); - preq_sig = (batch_request *)calloc(1, sizeof(batch_request)); - br_freed = FALSE; nanny = 1; strcpy(preq_sig->rq_ind.rq_signal.rq_jid, "1.napali"); preq_sig->rq_reply.brp_code = PBSE_UNKJOBID; post_job_delete_nanny(preq_sig); - fail_unless(br_freed == TRUE); } END_TEST @@ -416,7 +368,6 @@ Suite *req_delete_suite(void) suite_add_tcase(s, tc_core); tc_core = tcase_create("more"); - tcase_add_test(tc_core, test_duplicate_request); tcase_add_test(tc_core, test_handle_delete_all); tcase_add_test(tc_core, test_handle_single_delete); suite_add_tcase(s, tc_core); diff --git a/src/test/req_getcred/scaffolding.c b/src/test/req_getcred/scaffolding.c index 3938c2ba31..664524508f 100644 --- a/src/test/req_getcred/scaffolding.c +++ b/src/test/req_getcred/scaffolding.c @@ -70,3 +70,12 @@ unsigned long get_connectaddr(int sock, int mutex) void log_err(int errnum, const char *routine, const char *text) {} void log_record(int eventtype, int objclass, const char *objname, const char *text) {} void log_event(int eventtype, int objclass, const char *objname, const char *text) {} + +batch_request::batch_request() + { + } + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_getcred/test_uut.c b/src/test/req_getcred/test_uut.c index bda5966f11..6b4242783b 100644 --- a/src/test/req_getcred/test_uut.c +++ b/src/test/req_getcred/test_uut.c @@ -15,7 +15,6 @@ START_TEST(test_one) batch_request req; - memset(&req,0,sizeof(batch_request)); memset(svr_conn,0,sizeof(svr_conn)); req.rq_ind.rq_authen.rq_port = 42; strcpy(req.rq_user,"Tron"); diff --git a/src/test/req_holdjob/scaffolding.c b/src/test/req_holdjob/scaffolding.c index 3de92296c5..e08199f01c 100644 --- a/src/test/req_holdjob/scaffolding.c +++ b/src/test/req_holdjob/scaffolding.c @@ -161,3 +161,16 @@ int check_array_slot_limits(job *pjob, job_array *pa) { return(0); } + +batch_request::batch_request() + { + } + +batch_request::batch_request(const batch_request &other) + { + } + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_jobobit/scaffolding.c b/src/test/req_jobobit/scaffolding.c index c564206560..6b39bf339d 100644 --- a/src/test/req_jobobit/scaffolding.c +++ b/src/test/req_jobobit/scaffolding.c @@ -66,17 +66,6 @@ long disable_requeue = 0; completed_jobs_map_class completed_jobs_map; -struct batch_request *alloc_br(int type) - { - batch_request *preq; - if (alloc_br_null) - return(NULL); - - preq = (batch_request *)calloc(1, sizeof(batch_request)); - preq->rq_type = type; - - return(preq); - } char *parse_servername(const char *name, unsigned int *service) { @@ -209,7 +198,9 @@ job *svr_find_job(const char *jobid, int get_subjob) if (bad_job == 0) { - pjob = (job *)calloc(1, sizeof(job)); + pjob = new job(); + + memset(pjob->ji_wattr, 0, sizeof(pjob->ji_wattr)); strcpy(pjob->ji_qs.ji_jobid, jobid); pjob->ji_wattr[JOB_ATR_reported].at_flags = ATR_VFLAG_SET; @@ -267,7 +258,7 @@ int safe_strncat(char *str, const char *to_append, size_t space_remaining) batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -516,3 +507,15 @@ void job_array::update_array_values( { } +batch_request::batch_request(int type) : rq_type(type) + { + } + +batch_request::batch_request() + { + } + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_jobobit/test_uut.c b/src/test/req_jobobit/test_uut.c index 233d4b0d30..caf29c7795 100644 --- a/src/test/req_jobobit/test_uut.c +++ b/src/test/req_jobobit/test_uut.c @@ -206,7 +206,6 @@ START_TEST(return_stdfile_test) batch_request *p1 = &preq; batch_request *p2; - memset(&preq, 0, sizeof(preq)); pjob.ji_wattr[JOB_ATR_checkpoint_name].at_flags = ATR_VFLAG_SET; fail_unless(return_stdfile(&preq, &pjob, JOB_ATR_outpath) != NULL); @@ -278,8 +277,6 @@ START_TEST(setup_cpyfiles_test) { job *pjob = new job(); batch_request *preq; - alloc_br_null = 1; - fail_unless(setup_cpyfiles(NULL, pjob, strdup("from"), strdup("to"), 1, 1) == NULL); alloc_br_null = 0; strcpy(pjob->ji_qs.ji_jobid, "1.napali"); @@ -303,7 +300,7 @@ START_TEST(handle_returnstd_test) job *pjob; pjob = new job(); - preq = (batch_request *)calloc(1, sizeof(batch_request)); + preq = new batch_request(); strcpy(pjob->ji_qs.ji_jobid, "1.napali"); strcpy(pjob->ji_qs.ji_fileprefix, "1.napali"); diff --git a/src/test/req_manager/scaffolding.c b/src/test/req_manager/scaffolding.c index 36595e05e6..2859d50708 100644 --- a/src/test/req_manager/scaffolding.c +++ b/src/test/req_manager/scaffolding.c @@ -512,3 +512,7 @@ acl_special::acl_special() {} acl_special limited_acls; +batch_request::batch_request(int type) : rq_type(type) + { + } + diff --git a/src/test/req_message/scaffolding.c b/src/test/req_message/scaffolding.c index 6bbca748fb..a13ef03e05 100644 --- a/src/test/req_message/scaffolding.c +++ b/src/test/req_message/scaffolding.c @@ -68,7 +68,15 @@ int relay_to_mom(job **pjob_ptr, batch_request *request, void (*func)(struct w return(0); } -batch_request *duplicate_request(batch_request *preq, int type) +batch_request::batch_request(const batch_request &other) + { + } + +batch_request::batch_request() + { + } + +batch_request::~batch_request() + { - return(NULL); } diff --git a/src/test/req_modify/scaffolding.c b/src/test/req_modify/scaffolding.c index 49903af258..184a91287c 100644 --- a/src/test/req_modify/scaffolding.c +++ b/src/test/req_modify/scaffolding.c @@ -330,3 +330,22 @@ void job::set_plugin_resource_usage( } void update_slot_held_jobs(job_array *pa, int num_to_release) {} + +batch_request::batch_request(const batch_request &other) + { + } + +batch_request::batch_request(int type) + { + } + +void batch_request::update_object_id(int index) {} + +batch_request::batch_request() + { + } + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_quejob/scaffolding.c b/src/test/req_quejob/scaffolding.c index 0aa3869681..324b2c34d9 100644 --- a/src/test/req_quejob/scaffolding.c +++ b/src/test/req_quejob/scaffolding.c @@ -526,6 +526,13 @@ int req_runjob(batch_request *preq) return(0); } +batch_request::batch_request() {} +batch_request::batch_request(int type) : rq_type(type) {} + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_quejob/test_uut.c b/src/test/req_quejob/test_uut.c index fef4327be6..6e2f810084 100644 --- a/src/test/req_quejob/test_uut.c +++ b/src/test/req_quejob/test_uut.c @@ -103,12 +103,11 @@ END_TEST START_TEST(test_one) { - struct batch_request req; + batch_request req; job j; char cmd[500]; memset(&j,0,sizeof(j)); - memset(&req,0,sizeof(req)); fail_unless(req_jobcredential(&req) == PBSE_IVALREQ); @@ -120,17 +119,16 @@ START_TEST(test_one) fail_unless(req_jobcredential(&req) == PBSE_NONE); - memset(&req,0,sizeof(req)); - - strcpy(req.rq_ind.rq_jobfile.rq_jobid,"NotThisJob"); - fail_unless(req_jobscript(&req, false) == PBSE_IVALREQ); + batch_request req2; + strcpy(req2.rq_ind.rq_jobfile.rq_jobid,"NotThisJob"); + fail_unless(req_jobscript(&req2, false) == PBSE_IVALREQ); path_jobs = getcwd(path_to_jobs,sizeof(path_to_jobs)); strcat(path_jobs,"/"); sprintf(cmd,"rm -f %s*.SC",path_jobs); system(cmd); - strcpy(req.rq_ind.rq_jobfile.rq_jobid,"1.napali"); - fail_unless(req_jobscript(&req, false) == PBSE_NONE); + strcpy(req2.rq_ind.rq_jobfile.rq_jobid,"1.napali"); + fail_unless(req_jobscript(&req2, false) == PBSE_NONE); system(cmd); } END_TEST @@ -143,8 +141,6 @@ START_TEST(test_get_job_id) int created_here = 0; batch_request preq; - memset(&preq, 0, sizeof(preq)); - preq.rq_fromsvr = 1; strcpy(preq.rq_ind.rq_queuejob.rq_jid, "1.napali"); fail_unless(get_job_id(&preq, resc_access_perm, created_here, job_id) == PBSE_NONE); @@ -193,9 +189,7 @@ END_TEST START_TEST(test_req_commit) { - struct batch_request *preq; - - fail_unless((preq = (struct batch_request *)calloc(1, sizeof(struct batch_request))) != NULL); + struct batch_request *preq = new batch_request(); strcpy(preq->rq_ind.rq_commit, "1.napali"); default_queue = false; diff --git a/src/test/req_register/scaffolding.c b/src/test/req_register/scaffolding.c index 01c1d1fdae..fc7f93f3e0 100644 --- a/src/test/req_register/scaffolding.c +++ b/src/test/req_register/scaffolding.c @@ -167,7 +167,7 @@ void append_link(tlist_head *head, list_link *new_link, void *pobj) new_link->ll_prior->ll_next = new_link; /* now visible to forward iteration */ } -int issue_to_svr(const char *servern, struct batch_request **preq, void (*replyfunc)(struct work_task *)) +int issue_to_svr(const char *servern, batch_request *preq, void (*replyfunc)(struct work_task *)) { fprintf(stderr, "The call to issue_to_svr to be mocked!!\n"); exit(1); @@ -308,7 +308,7 @@ int unlock_queue(struct pbs_queue *the_queue, const char *id, const char *msg, i batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -396,3 +396,18 @@ job_array::job_array() : job_ids(NULL), jobs_recovered(0), ai_ghost_recovered(fa { } + +batch_request::~batch_request() + + { + } + +batch_request::batch_request() + + { + } + +batch_request::batch_request(int type) : rq_type(type) + + { + } diff --git a/src/test/req_register/test_uut.c b/src/test/req_register/test_uut.c index 92c27341e6..a6e3997490 100644 --- a/src/test/req_register/test_uut.c +++ b/src/test/req_register/test_uut.c @@ -92,7 +92,7 @@ END_TEST START_TEST(set_array_depend_holds_test) { - batch_request *preq = (batch_request *)calloc(1, sizeof(batch_request)); + batch_request *preq = new batch_request(); job_array *pa = new job_array(); strcpy(preq->rq_ind.rq_register.rq_child, job1); @@ -111,7 +111,7 @@ START_TEST(check_dependency_job_test) batch_request preq; char *jobid = strdup("1.napali"); - memset(&preq, 0, sizeof(preq)); + preq.rq_fromsvr = 0; fail_unless(check_dependency_job(NULL, NULL, NULL) == PBSE_BAD_PARAMETER, "passed bad params?"); fail_unless(check_dependency_job(jobid, &preq, &pjob) == PBSE_IVALREQ, "accepted non-server request?"); @@ -127,7 +127,8 @@ START_TEST(check_dependency_job_test) fail_unless(check_dependency_job((char *)"bob", &preq, &pjob) == PBSE_JOBNOTFOUND, "wrong rc"); fail_unless(pjob == NULL, "didn't set job to NULL"); svr = 2; - + + preq.rq_ind.rq_register.rq_dependtype = JOB_DEPEND_TYPE_AFTERANY; fail_unless(check_dependency_job((char *)"2.napali", &preq, &pjob) == PBSE_BADSTATE, "wrong rc"); } END_TEST @@ -280,7 +281,6 @@ START_TEST(register_dep_test) int made = 0; initialize_depend_attr(&pattr); - memset(&preq, 0, sizeof(preq)); strcpy(preq.rq_ind.rq_register.rq_svr, host); strcpy(preq.rq_ind.rq_register.rq_child, job1); @@ -300,7 +300,6 @@ START_TEST(unregister_dep_test) struct depend *pdep; batch_request preq; - memset(&preq, 0, sizeof(preq)); strcpy(preq.rq_ind.rq_register.rq_svr, host); strcpy(preq.rq_ind.rq_register.rq_child, job1); preq.rq_ind.rq_register.rq_dependtype = 1; @@ -428,7 +427,7 @@ END_TEST START_TEST(req_register_test) { - batch_request *preq = (batch_request *)calloc(1, sizeof(batch_request)); + batch_request *preq = new batch_request(); strcpy(preq->rq_ind.rq_register.rq_parent, job1); preq->rq_fromsvr = 1; @@ -557,7 +556,6 @@ START_TEST(register_before_dep_test) int rc = 0; char buf[1000]; - memset(&preq, 0, sizeof(preq)); memset(&pjob, 0, sizeof(pjob)); strcpy(preq.rq_ind.rq_register.rq_owner, "dbeer"); @@ -587,7 +585,6 @@ START_TEST(register_dependency_test) job pjob; pbs_attribute *pattr; - memset(&preq, 0, sizeof(preq)); memset(&pjob, 0, sizeof(pjob)); strcpy(preq.rq_ind.rq_register.rq_owner, "dbeer"); @@ -614,7 +611,6 @@ START_TEST(release_before_dependency_test) pbs_attribute *pattr; struct depend *pdep; - memset(&preq, 0, sizeof(preq)); memset(&pjob, 0, sizeof(pjob)); strcpy(preq.rq_ind.rq_register.rq_owner, "dbeer"); @@ -640,7 +636,6 @@ START_TEST(release_syncwith_dependency_test) struct depend *pdep; batch_request preq; - memset(&preq, 0, sizeof(preq)); memset(&pjob, 0, sizeof(pjob)); pattr = &pjob.ji_wattr[JOB_ATR_depend]; initialize_depend_attr(pattr); @@ -706,7 +701,6 @@ START_TEST(delete_dependency_job_test) job *pjob = job_alloc(); batch_request preq; - memset(&preq, 0, sizeof(preq)); strcpy(preq.rq_ind.rq_register.rq_parent, job1); strcpy(preq.rq_ind.rq_register.rq_child, job1); diff --git a/src/test/req_rerun/scaffolding.c b/src/test/req_rerun/scaffolding.c index 262007e1e7..d678ca2a90 100644 --- a/src/test/req_rerun/scaffolding.c +++ b/src/test/req_rerun/scaffolding.c @@ -96,7 +96,7 @@ job *chk_job_request(char *jobid, struct batch_request *preq) batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -182,3 +182,17 @@ batch_request *duplicate_request( { return(NULL); } + +batch_request::batch_request(const batch_request &other) + { + } + +batch_request::batch_request() + { + } + +batch_request::~batch_request() + + { + } + diff --git a/src/test/req_rerun/test_uut.c b/src/test/req_rerun/test_uut.c index 8451848ae1..7fe37d78eb 100644 --- a/src/test/req_rerun/test_uut.c +++ b/src/test/req_rerun/test_uut.c @@ -35,8 +35,6 @@ START_TEST(test_handle_requeue_all) { batch_request preq; - memset(&preq, 0, sizeof(preq)); - // fail due to lack of permissions fail_unless(handle_requeue_all(&preq) == PBSE_PERM); diff --git a/src/test/req_runjob/scaffolding.c b/src/test/req_runjob/scaffolding.c index e8b056e00c..f95b009472 100644 --- a/src/test/req_runjob/scaffolding.c +++ b/src/test/req_runjob/scaffolding.c @@ -403,3 +403,16 @@ void job_array::update_array_values( { } +batch_request::~batch_request() + + { + } + +batch_request::batch_request(const batch_request &other) + + { + } + +batch_request::batch_request() + { + } diff --git a/src/test/req_runjob/test_uut.c b/src/test/req_runjob/test_uut.c index c6434ade0c..af94cfd829 100644 --- a/src/test/req_runjob/test_uut.c +++ b/src/test/req_runjob/test_uut.c @@ -38,7 +38,6 @@ START_TEST(test_get_mail_text) const char *txt = "tom is bob"; char *ret_txt; - memset(&preq, 0, sizeof(preq)); preq.rq_reply.brp_un.brp_txt.brp_txtlen = 999000001; preq.rq_reply.brp_un.brp_txt.brp_str = strdup(txt); fail_unless(get_mail_text(&preq, "1.napali") == NULL); @@ -54,7 +53,6 @@ START_TEST(test_two) { struct batch_request request; job myjob; - memset(&request, 0, sizeof(struct batch_request)); memset(&myjob, 0, sizeof(job)); myjob.ji_qs.ji_state = JOB_STATE_RUNNING; myjob.ji_qs.ji_un.ji_exect.ji_momaddr = 167838724; diff --git a/src/test/req_signal/scaffolding.c b/src/test/req_signal/scaffolding.c index a642f3ef77..859c4e1f66 100644 --- a/src/test/req_signal/scaffolding.c +++ b/src/test/req_signal/scaffolding.c @@ -183,4 +183,20 @@ void rel_resc( return; } +batch_request::~batch_request() + { + } + +batch_request::batch_request(const batch_request &other) + + { + } + +batch_request::batch_request() + { + } + +batch_request::batch_request(int type) : rq_type(type) + { + } diff --git a/src/test/req_signal/test_uut.c b/src/test/req_signal/test_uut.c index 145db42271..5e25b4926b 100644 --- a/src/test/req_signal/test_uut.c +++ b/src/test/req_signal/test_uut.c @@ -20,7 +20,6 @@ START_TEST(test_req_signaljob_relaying_msg) { struct batch_request request; job myjob; - memset(&request, 0, sizeof(struct batch_request)); memset(&myjob, 0, sizeof(job)); myjob.ji_qs.ji_state = JOB_STATE_RUNNING; myjob.ji_qs.ji_un.ji_exect.ji_momaddr = 167838724; diff --git a/src/test/req_stat/scaffolding.c b/src/test/req_stat/scaffolding.c index 889041764b..789dcda60e 100644 --- a/src/test/req_stat/scaffolding.c +++ b/src/test/req_stat/scaffolding.c @@ -368,3 +368,21 @@ bool pbsnode::hasprop(std::vector *needed) const { preply->brp_choice = type; } + +batch_request::~batch_request() + + { + } + +batch_request::batch_request(const batch_request &other) + + { + } + +batch_request::batch_request() + { + } + +batch_request::batch_request(int type) : rq_type(type) + { + } diff --git a/src/test/req_stat/test_uut.c b/src/test/req_stat/test_uut.c index 9c062ff0ed..d95a2dcef1 100644 --- a/src/test/req_stat/test_uut.c +++ b/src/test/req_stat/test_uut.c @@ -47,7 +47,6 @@ START_TEST(test_stat_update) batch_request preq; stat_cntl cntl; - memset(&preq, 0, sizeof(preq)); memset(&cntl, 0, sizeof(cntl)); abort_called = 0; diff --git a/src/test/req_track/scaffolding.c b/src/test/req_track/scaffolding.c index 86ac86018e..c75db9f900 100644 --- a/src/test/req_track/scaffolding.c +++ b/src/test/req_track/scaffolding.c @@ -87,3 +87,17 @@ mutex_mgr::~mutex_mgr() { } +batch_request::~batch_request() + + { + } + +batch_request::batch_request(const batch_request &other) + + { + } + +batch_request::batch_request() + { + } + diff --git a/src/test/requests/scaffolding.c b/src/test/requests/scaffolding.c index 0344ef28a7..6b057706f3 100644 --- a/src/test/requests/scaffolding.c +++ b/src/test/requests/scaffolding.c @@ -536,3 +536,5 @@ int send_job_obit(job *pjob, int exit_status) { return(PBSE_NONE); } + +batch_request::batch_request(int type) : rq_type(type) {} From 7f5ce6b60a4479086f55116b68f1fd8d56a6510c Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 14 Nov 2016 11:39:15 -0700 Subject: [PATCH 055/312] Make torque-build properly recognize SLES 12SP1. --- buildutils/torque-build | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index fa71c500b3..6b59cd8886 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -289,8 +289,10 @@ then mkdir -p buildutils/rpmbuild/SOURCES mkdir -p buildutils/rpmbuild/SPECS mkdir -p buildutils/rpmbuild/RPMS/{i386,i686,noarch,x86_64} - # get the distro - distro=$(buildutils/determine.distro -r) + + # ignore SP version suffix for SLES12 + echo "${distro}" | fgrep -q sles12. && distro=sles12 + if [ "${distro}" = el7 ]; then cp buildutils/torque.adaptive.spec.el7 \ buildutils/rpmbuild/SPECS/torque.adaptive.spec From 21f3806e49972e12a6dea7a788b7de247360475d Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 14 Nov 2016 11:48:13 -0700 Subject: [PATCH 056/312] Fix two potential memory leaks in the batch request change. --- src/server/req_register.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server/req_register.c b/src/server/req_register.c index 40d57b1ed6..a78339f868 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -2436,6 +2436,7 @@ int send_depend_req( { if (pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str == NULL) { + delete preq; return(PBSE_BADATVAL); } @@ -2502,6 +2503,9 @@ int send_depend_req( if (rc != PBSE_NONE) { + if (preq != NULL) + delete preq; + sprintf(log_buf, "Unable to perform dependency with job %s\n", pparent->dc_child.c_str()); log_err(rc, __func__, log_buf); From 3635ffc0eee991539b5efadd592a6d56285bb0ca Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 14 Nov 2016 13:51:57 -0700 Subject: [PATCH 057/312] Fix some scaffolding issues. --- src/test/job_recov/scaffolding.c | 1 - src/test/req_deletearray/scaffolding.c | 2 +- src/test/req_gpuctrl/scaffolding.c | 2 +- src/test/req_holdjob/scaffolding.c | 2 +- src/test/req_message/scaffolding.c | 2 +- src/test/req_modify/scaffolding.c | 2 +- src/test/req_runjob/scaffolding.c | 2 +- src/test/req_shutdown/scaffolding.c | 2 +- src/test/req_signal/scaffolding.c | 2 +- 9 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/test/job_recov/scaffolding.c b/src/test/job_recov/scaffolding.c index cd4a8b0f03..a6fe72134a 100644 --- a/src/test/job_recov/scaffolding.c +++ b/src/test/job_recov/scaffolding.c @@ -285,7 +285,6 @@ int decode_b(pbs_attribute *patr, const char *name, const char *rn, const char int nextjobnum_chk(pbs_attribute *pattr, void *pobject, int actmode) {return 0;} int svr_chk_owner(struct batch_request *preq, job *pjob) {return 0;} int comp_checkpoint(pbs_attribute *attr, pbs_attribute *with) {return 0;} -batch_request *get_remove_batch_request(char *br_id) {return NULL;} long calc_job_cost(job *pjob) {return(0);} int issue_to_svr(const char *servern, batch_request *preq, void (*replyfunc)(struct work_task *)) diff --git a/src/test/req_deletearray/scaffolding.c b/src/test/req_deletearray/scaffolding.c index 116a11bdef..00f6a39c77 100644 --- a/src/test/req_deletearray/scaffolding.c +++ b/src/test/req_deletearray/scaffolding.c @@ -153,7 +153,7 @@ int array_delete( batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); diff --git a/src/test/req_gpuctrl/scaffolding.c b/src/test/req_gpuctrl/scaffolding.c index d92c16dade..a51192b866 100644 --- a/src/test/req_gpuctrl/scaffolding.c +++ b/src/test/req_gpuctrl/scaffolding.c @@ -15,7 +15,7 @@ void req_reject(int code, int aux, struct batch_request *preq, const char *HostN batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); diff --git a/src/test/req_holdjob/scaffolding.c b/src/test/req_holdjob/scaffolding.c index e08199f01c..70c3b83c42 100644 --- a/src/test/req_holdjob/scaffolding.c +++ b/src/test/req_holdjob/scaffolding.c @@ -119,7 +119,7 @@ int copy_batchrequest(struct batch_request **newreq, struct batch_request *preq, batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); diff --git a/src/test/req_message/scaffolding.c b/src/test/req_message/scaffolding.c index a13ef03e05..a18fbeb40c 100644 --- a/src/test/req_message/scaffolding.c +++ b/src/test/req_message/scaffolding.c @@ -42,7 +42,7 @@ void free_br(struct batch_request *preq) {} batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); diff --git a/src/test/req_modify/scaffolding.c b/src/test/req_modify/scaffolding.c index 184a91287c..ac17cb4a9d 100644 --- a/src/test/req_modify/scaffolding.c +++ b/src/test/req_modify/scaffolding.c @@ -218,7 +218,7 @@ int get_svr_attr_b(int index, bool *b) batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); diff --git a/src/test/req_runjob/scaffolding.c b/src/test/req_runjob/scaffolding.c index f95b009472..b0c1824243 100644 --- a/src/test/req_runjob/scaffolding.c +++ b/src/test/req_runjob/scaffolding.c @@ -302,7 +302,7 @@ int unlock_queue(struct pbs_queue *the_queue, const char *id, const char *msg, i batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); diff --git a/src/test/req_shutdown/scaffolding.c b/src/test/req_shutdown/scaffolding.c index 9af49c6f57..9b42f3353d 100644 --- a/src/test/req_shutdown/scaffolding.c +++ b/src/test/req_shutdown/scaffolding.c @@ -116,7 +116,7 @@ int unlock_queue(struct pbs_queue *the_queue, const char *id, const char *msg, i batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); diff --git a/src/test/req_signal/scaffolding.c b/src/test/req_signal/scaffolding.c index 859c4e1f66..d79a191415 100644 --- a/src/test/req_signal/scaffolding.c +++ b/src/test/req_signal/scaffolding.c @@ -97,7 +97,7 @@ void free_br(struct batch_request *b) {} batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); From 383fcf6ad076ee35bf1f51e6abce974cf202310c Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 14 Nov 2016 14:33:11 -0700 Subject: [PATCH 058/312] Fix two test errors. --- src/test/req_register/scaffolding.c | 1 + src/test/req_rerun/scaffolding.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/req_register/scaffolding.c b/src/test/req_register/scaffolding.c index fc7f93f3e0..0fd27f9b0f 100644 --- a/src/test/req_register/scaffolding.c +++ b/src/test/req_register/scaffolding.c @@ -377,6 +377,7 @@ int is_svr_attr_set(int index) job::job() : ji_rejectdest() { + this->ji_wattr[JOB_ATR_hold].at_flags = 0; } job::~job() {} diff --git a/src/test/req_rerun/scaffolding.c b/src/test/req_rerun/scaffolding.c index d678ca2a90..5e90a95cc7 100644 --- a/src/test/req_rerun/scaffolding.c +++ b/src/test/req_rerun/scaffolding.c @@ -187,7 +187,7 @@ batch_request::batch_request(const batch_request &other) { } -batch_request::batch_request() +batch_request::batch_request() : rq_perm(0), rq_fromsvr(0) { } From 5119cf3a2050c7d2567f634928c5dad8f0d2ce38 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 14 Nov 2016 14:53:18 -0700 Subject: [PATCH 059/312] Fix another unit test initialization issue. --- src/test/req_signal/scaffolding.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/req_signal/scaffolding.c b/src/test/req_signal/scaffolding.c index d79a191415..70cefb9d1a 100644 --- a/src/test/req_signal/scaffolding.c +++ b/src/test/req_signal/scaffolding.c @@ -154,7 +154,11 @@ char * netaddr_long(long ap, char *out) return(NULL); } -job::job() {} +job::job() + { + memset(&this->ji_qs, 0, sizeof(this->ji_qs)); + } + job::~job() {} int svr_setjobstate( From d91ce7e2b183a5c16dc46965470011a48ffbff38 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 14 Nov 2016 15:09:15 -0700 Subject: [PATCH 060/312] Fix another scaffolding issue. --- src/test/req_track/scaffolding.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/req_track/scaffolding.c b/src/test/req_track/scaffolding.c index c75db9f900..4d1f7d06b8 100644 --- a/src/test/req_track/scaffolding.c +++ b/src/test/req_track/scaffolding.c @@ -52,10 +52,9 @@ void release_req(struct work_task *pwt) exit(1); } -int issue_to_svr(const char *servern, struct batch_request **preq, void (*replyfunc)(struct work_task *)) +int issue_to_svr(const char *servern, batch_request *preq, void (*replyfunc)(struct work_task *)) { - fprintf(stderr, "The call to issue_to_svr to be mocked!!\n"); - exit(1); + return(0); } void free_br(batch_request *preq) {} @@ -101,3 +100,7 @@ batch_request::batch_request() { } +batch_request::batch_request(int type) : rq_type(type) + { + } + From 44291abe740103d2b4a8b1a9d264a5075ded4a00 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 14 Nov 2016 16:44:57 -0700 Subject: [PATCH 061/312] Fix a Coverity complaint. --- src/server/req_register.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/server/req_register.c b/src/server/req_register.c index a78339f868..35711f7091 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -3380,31 +3380,28 @@ void removeBeforeAnyDependencies( job *pLockedJob = *pjob_ptr; std::string jobid(pLockedJob->ji_qs.ji_jobid); - if (pLockedJob != NULL) + mutex_mgr job_mutex(pLockedJob->ji_mutex,true); + pbs_attribute *pattr = &pLockedJob->ji_wattr[JOB_ATR_depend]; + + struct depend *pDep = find_depend(JOB_DEPEND_TYPE_BEFOREANY,pattr); + if (pDep != NULL) { - mutex_mgr job_mutex(pLockedJob->ji_mutex,true); - pbs_attribute *pattr = &pLockedJob->ji_wattr[JOB_ATR_depend]; - struct depend *pDep = find_depend(JOB_DEPEND_TYPE_BEFOREANY,pattr); - if (pDep != NULL) + unsigned int dp_jobs_size = pDep->dp_jobs.size(); + for (unsigned int i = 0; i < dp_jobs_size; i++) { - - unsigned int dp_jobs_size = pDep->dp_jobs.size(); - for (unsigned int i = 0; i < dp_jobs_size; i++) + depend_job *pDepJob = pDep->dp_jobs[i]; + std::string depID(pDepJob->dc_child); + job_mutex.unlock(); + removeAfterAnyDependency(depID.c_str(), jobid.c_str()); + pLockedJob = svr_find_job(jobid.c_str(), FALSE); + if (pLockedJob == NULL) { - depend_job *pDepJob = pDep->dp_jobs[i]; - std::string depID(pDepJob->dc_child); - job_mutex.unlock(); - removeAfterAnyDependency(depID.c_str(), jobid.c_str()); - pLockedJob = svr_find_job(jobid.c_str(), FALSE); - if (pLockedJob == NULL) - { - *pjob_ptr = NULL; - break; - } - - job_mutex.mark_as_locked(); + *pjob_ptr = NULL; + break; } + + job_mutex.mark_as_locked(); } } From 766558a435f455f18ac54760caacfb3e117f81b5 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 17 Nov 2016 13:48:57 -0700 Subject: [PATCH 062/312] Fix a regression around the conversion of batch_request to a class. --- src/server/req_signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/req_signal.c b/src/server/req_signal.c index 3c8f888b49..4ae4a792e2 100644 --- a/src/server/req_signal.c +++ b/src/server/req_signal.c @@ -251,7 +251,7 @@ int issue_signal( { int rc; job *pjob = *pjob_ptr; - batch_request newreq; + batch_request newreq(PBS_BATCH_SignalJob); char jobid[PBS_MAXSVRJOBID + 1]; newreq.rq_extra = extra; From fc6ccda73f6088b046da6e9fc8a625559f59e503 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 17 Nov 2016 15:27:24 -0700 Subject: [PATCH 063/312] TRQ-3667. Escape the spaces in a job's file path. --- src/resmom/start_exec.c | 27 +++++++++++++++++++++++++-- src/test/start_exec/test_uut.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index 7ef76a7111..46ac0991f6 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -9637,10 +9637,31 @@ void add_wkm_end( return; } /* END add_wkm_end() */ +#endif /* ENABLE_CSA */ -#endif /* ENABLE_CSA */ +void escape_spaces( + + const char *str, + std::string &escaped) + + { + if (str != NULL) + { + const char *ptr = str; + + while (*ptr != '\0') + { + if (*ptr == ' ') + escaped += '\\'; + + escaped += *ptr++; + } + } + } // escape_spaces() + + /* * @param pjob - used to set up the user's environment if desired @@ -9693,9 +9714,11 @@ int expand_path( environ = vtable.v_envp; } + std::string escaped; + escape_spaces(path_in, escaped); /* expand the path */ - switch (wordexp(path_in, &exp, WRDE_NOCMD | WRDE_UNDEF)) + switch (wordexp(escaped.c_str(), &exp, WRDE_NOCMD | WRDE_UNDEF)) { case 0: diff --git a/src/test/start_exec/test_uut.c b/src/test/start_exec/test_uut.c index 9daa38ae24..2dc68c687b 100644 --- a/src/test/start_exec/test_uut.c +++ b/src/test/start_exec/test_uut.c @@ -47,6 +47,7 @@ void exec_bail(job *pjob, int code, std::set *sisters_contacted); int read_launcher_child_status(struct startjob_rtn *sjr, const char *job_id, int parent_read, int parent_write); int process_launcher_child_status(struct startjob_rtn *sjr, const char *job_id, const char *application_name); void update_task_and_job_states_after_launch(task *ptask, job *pjob, const char *application_name); +void escape_spaces(const char *str, std::string &escaped); #ifdef PENABLE_LINUX_CGROUPS unsigned long long get_memory_limit_from_resource_list(job *pjob); @@ -139,6 +140,34 @@ START_TEST(test_read_launcher_child_status) fail_unless(read_launcher_child_status(&srj, jobid, parent_read, parent_write) != PBSE_NONE); } END_TEST + + +START_TEST(test_escape_spaces) + { + std::string escaped; + + const char *ws1 = "one two"; + const char *ws2 = "one two three"; + const char *ws3 = "one two three four"; + + escape_spaces(ws1, escaped); + // should have added a slash + fail_unless(escaped.size() == strlen(ws1) + 1); + fail_unless(escaped == "one\\ two"); + + escaped.clear(); + escape_spaces(ws2, escaped); + // should have added two slashes + fail_unless(escaped.size() == strlen(ws2) + 2); + fail_unless(escaped == "one\\ two\\ three"); + + escaped.clear(); + escape_spaces(ws3, escaped); + // should have added three slashes + fail_unless(escaped.size() == strlen(ws3) + 3); + fail_unless(escaped == "one\\ two\\ three\\ four"); + } +END_TEST #ifdef PENABLE_LINUX_CGROUPS @@ -643,6 +672,7 @@ Suite *start_exec_suite(void) tcase_add_test(tc_core, test_get_num_nodes_ppn); tcase_add_test(tc_core, test_setup_process_launch_pipes); tcase_add_test(tc_core, test_read_launcher_child_status); + tcase_add_test(tc_core, test_escape_spaces); #ifdef PENABLE_LINUX_CGROUPS tcase_add_test(tc_core, get_memory_limit_from_resource_list_test); #endif From f848ac8773230297f321df03b6280dda6bb690d5 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 18 Nov 2016 11:46:37 -0700 Subject: [PATCH 064/312] Fix a race-condition in cleaning up jobs. --- src/include/pbs_job.h | 2 +- src/server/completed_jobs_map.cpp | 29 +++++------------------ src/server/job_func.c | 13 ++-------- src/test/completed_jobs_map/scaffolding.c | 3 +-- 4 files changed, 10 insertions(+), 37 deletions(-) diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 56f849fba5..7452abe56f 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -1307,7 +1307,7 @@ job *job_clone(job *,struct job_array *, int, bool); job *svr_find_job(const char *jobid, int get_subjob); job *svr_find_job_by_id(int internal_job_id); job *find_job_by_array(all_jobs *aj, const char *job_id, int get_subjob, bool locked); -bool job_id_exists(const std::string &job_id_string, int *rcode); +bool job_id_exists(const std::string &job_id_string); bool internal_job_id_exists(int internal_id); #else extern job *mom_find_job(const char *); diff --git a/src/server/completed_jobs_map.cpp b/src/server/completed_jobs_map.cpp index 7e50a71bb9..6d22a5fcb8 100644 --- a/src/server/completed_jobs_map.cpp +++ b/src/server/completed_jobs_map.cpp @@ -165,36 +165,19 @@ int completed_jobs_map_class::cleanup_completed_jobs( it != completed_jobs_map.end(); ++it) { - bool rc = false; - // if cleanup time is before/at now, then clean up if (it->second <= now) { - int retcode = PBSE_NONE; work_task *pnew; - // Does job exist - - while((rc = job_id_exists(it->first, &retcode)) == false) + // Does job still exist + if (job_id_exists(it->first) == false) { - if (retcode != 0) - { - // Someone else has a lock we want. give ours up and try again - pthread_mutex_unlock(&completed_jobs_map_mutex); - pthread_mutex_lock(&completed_jobs_map_mutex); - continue; - } - else - break; - } + // Job is gone, mark it for removal + to_remove.push_back(it->first); - if (rc == false) - { - // Job is gone, mark it for removal - to_remove.push_back(it->first); - - continue; - } + continue; + } // create a work task struct to be passed to handle_complete_second_time() if ((pnew = (struct work_task *)calloc(1, sizeof(struct work_task))) == NULL) diff --git a/src/server/job_func.c b/src/server/job_func.c index b448fa23ec..67fb86a9cb 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -2428,21 +2428,12 @@ int split_job( bool job_id_exists( - const std::string &job_id_string, - int *rcode) + const std::string &job_id_string) { - int ret; bool rc = false; - ret = alljobs.trylock(); - if (ret != 0) - { - *rcode = ret; - return(false); - } - - *rcode = ret; + alljobs.lock(); if (alljobs.find(job_id_string) != NULL) { diff --git a/src/test/completed_jobs_map/scaffolding.c b/src/test/completed_jobs_map/scaffolding.c index d1f6be84ea..b38abed049 100644 --- a/src/test/completed_jobs_map/scaffolding.c +++ b/src/test/completed_jobs_map/scaffolding.c @@ -31,8 +31,7 @@ void *handle_complete_second_time(work_task *wt) bool job_id_exists( - const std::string &job_id_string, - int *rcode) + const std::string &job_id_string) { return(job_id_exists_rc); From 06664a7c9964fba1b3665e44e64ae9b4a05243d6 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 18 Nov 2016 11:47:25 -0700 Subject: [PATCH 065/312] TRQ-3682. Fix gpu indexing for multi-gpu nodes. Also, if cgroups are enabled, don't set $CUDA_VISIBLE_DEVICES unless requested since it is redundant. --- src/resmom/nvidia.c | 115 +++++++++++++++++++++++++++++++++ src/resmom/parse_config.c | 8 +++ src/resmom/start_exec.c | 30 +++++++-- src/test/start_exec/test_uut.c | 31 +++++++-- 4 files changed, 172 insertions(+), 12 deletions(-) diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index b4a3285ac6..c03f21b3f8 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -151,6 +151,14 @@ extern nodeboard node_boards[]; #endif +// Default to version 7.0 +extern int cuda_version; +std::string nvcc_version_cmd("/usr/local/cuda/bin/nvcc --version"); + +// Forward declaration +void set_cuda_version(); + + /* * Function to log nvml errors */ @@ -485,6 +493,9 @@ int check_nvidia_setup() nvidia_setup_is_ok = TRUE; #endif /* NVML_API */ } + + set_cuda_version(); + return (nvidia_setup_is_ok); } @@ -2202,3 +2213,107 @@ int add_gpu_status( return(PBSE_NONE); } /* END add_gpu_status() */ + + + +/* + * matches_version_string() + * + * Sets the global variable cuda_version from versiong_string, if possible + * + * @param version_string - a string that should match the format number.number.number + * @return true if the string matched the appropriate format and the version was set + */ + +bool matches_version_string( + + char *version_string) + + { + char *ptr = version_string; + char *end = ptr; + int tmp_version; + bool match = false; + + tmp_version += strtol(ptr, &end, 10) * 10; + + if (*end == '.') + { + ptr = end + 1; + + tmp_version += strtol(ptr, &end, 10); + + if (*end == '.') + { + // valid format + match = true; + cuda_version = tmp_version; + } + } + + return(match); + } // END matches_version_string() + + + +/* + * set_cuda_version() + * + * Determines what version of CUDA is installed on this machine + */ + +void set_cuda_version() + { + char buf[MAXLINE * 4]; + FILE *nvcc_pipe; + char *ptr = buf; + int bytes_read = 0; + bool version_set = false; + + if ((nvcc_pipe = popen(nvcc_version_cmd.c_str(), "r")) == NULL) + { + snprintf(log_buffer, sizeof(log_buffer), + "Cannot open pipe to get CUDA version with command '%s' - assuming version %d", + nvcc_version_cmd.c_str(), cuda_version); + log_err(errno, __func__, log_buffer); + } + + memset(buf, 0, sizeof(buf)); + int fd = fileno(nvcc_pipe); + + while ((version_set == false) && + ((bytes_read = read(fd, ptr, sizeof(buf) - 1)) > 0)) + { + char *tmpptr = ptr; + + // Version string appears in the format V.., so find the V + // and check if the rest matches + while ((tmpptr = strrchr(tmpptr, 'V')) != NULL) + { + if (matches_version_string(tmpptr + 1) == true) + { + version_set = true; + break; + } + + // At the beginning + if (tmpptr == ptr) + break; + else + tmpptr--; + } + } + + pclose(nvcc_pipe); + + if (version_set == false) + { + snprintf(log_buffer, sizeof(log_buffer), + "Couldn't find a CUDA version string with command '%s', defaulting to version %d", + nvcc_version_cmd.c_str(), cuda_version); + log_err(-1, __func__, log_buffer); + } + + } // END set_cuda_version() + + diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index beaf5621a5..d2920cbc3a 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -165,7 +165,13 @@ char **maskclient = NULL; /* wildcard connections */ char MOMConfigVersion[64]; int MOMConfigDownOnError = 0; int MOMConfigRestart = 0; +// Setting this variable is redundant for cgroups, so we'll default to false +// It can only cause us pain. +#ifdef PENABLE_LINUX_CGROUPS +int MOMCudaVisibleDevices = 0; +#else int MOMCudaVisibleDevices = 1; +#endif double wallfactor = 1.00; std::vector pcphosts; long pe_alarm_time = PBS_PROLOG_TIME; @@ -3304,6 +3310,8 @@ const char *validuser( return("no"); } /* END validuser() */ + + u_long setcudavisibledevices( const char *value) /* I */ diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index 46ac0991f6..a4afa8e93a 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -252,6 +252,9 @@ extern int numa_index; extern int linux_time; +// Default to version 7.0 +int cuda_version = 70; + /* Local Variables */ static int script_in; /* script file, will be stdin */ @@ -1331,7 +1334,6 @@ int TMakeTmpDir( - /* * get_indices_from_exec_str * @@ -1348,13 +1350,17 @@ int TMakeTmpDir( * @param exec_str - the string we're parsing * @param buf - where the list of absolute mic indices goes * @param buf_size - maximum size that can be written into buf + * @param cuda_version - -1 if we're not doing gpus, or the version of cuda otherwise + * If we are doing cuda, and we're above version 7.0, we ignore the index + * in the string and count from 0. */ int get_indices_from_exec_str( - const char *exec_str, /* I */ - char *buf, /* O */ - int buf_size) /* I */ + const char *exec_str, + char *buf, + int buf_size, + int cuda_version) { char *work_str; @@ -1367,6 +1373,7 @@ int get_indices_from_exec_str( #endif int numa_offset = 0; int index; + int relative_count = 0; if ((buf == NULL) || (exec_str == NULL)) @@ -1408,12 +1415,19 @@ int get_indices_from_exec_str( if ((slash = strchr(tok, '/')) != NULL) { - index = strtol(slash+1, NULL, 10) + numa_offset; + // CUDA versions 7.0 and greater always index from 0 on up for the program, so if you + // have gpus 2 and 3, we should pass 0 and 1 + if (cuda_version >= 70) + index = relative_count; + else + index = strtol(slash+1, NULL, 10) + numa_offset; if (buf[0] != '\0') snprintf(buf + strlen(buf), buf_size - strlen(buf), ",%d", index); else snprintf(buf, buf_size, "%d", index); + + relative_count++; } } @@ -1427,6 +1441,7 @@ int get_indices_from_exec_str( } /* END get_indices_from_exec_str() */ + /* * get_num_nodes_ppn * @@ -1745,7 +1760,7 @@ int InitUserEnv( if (pjob->ji_wattr[JOB_ATR_exec_mics].at_val.at_str != NULL) { - get_indices_from_exec_str(pjob->ji_wattr[JOB_ATR_exec_mics].at_val.at_str, buf, sizeof(buf)); + get_indices_from_exec_str(pjob->ji_wattr[JOB_ATR_exec_mics].at_val.at_str, buf, sizeof(buf), -1); bld_env_variables(&vtable, variables_else[tveOffloadDevices], buf); } @@ -1753,7 +1768,8 @@ int InitUserEnv( { if (MOMCudaVisibleDevices) { - get_indices_from_exec_str(pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str, buf, sizeof(buf)); + get_indices_from_exec_str(pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str, buf, sizeof(buf), + cuda_version); bld_env_variables(&vtable, variables_else[tveCudaVisibleDevices], buf); } } diff --git a/src/test/start_exec/test_uut.c b/src/test/start_exec/test_uut.c index 2dc68c687b..fdf57b802d 100644 --- a/src/test/start_exec/test_uut.c +++ b/src/test/start_exec/test_uut.c @@ -15,7 +15,7 @@ #include "test_uut.h" int job_nodes(job &pjob); -int get_indices_from_exec_str(const char *exec_str, char *buf, int buf_size); +int get_indices_from_exec_str(const char *exec_str, char *buf, int buf_size, int cuda_version); int remove_leading_hostname(char **jobpath); int get_num_nodes_ppn(const char*, int*, int*); int setup_process_launch_pipes(int &kid_read, int &kid_write, int &parent_read, int &parent_write); @@ -495,19 +495,40 @@ START_TEST(test_get_indices_from_exec_str) { char buf[1024]; - fail_unless(get_indices_from_exec_str(NULL, NULL, 0) != PBSE_NONE); + fail_unless(get_indices_from_exec_str(NULL, NULL, 0, -1) != PBSE_NONE); strcpy(mom_alias, "slesmic"); - fail_unless(get_indices_from_exec_str("slesmic-0-mic/1+slesmic-0-mic/0", buf, sizeof(buf)) == PBSE_NONE); + fail_unless(get_indices_from_exec_str("slesmic-0-mic/1+slesmic-0-mic/0", + buf, + sizeof(buf), + -1) == PBSE_NONE); fail_unless(!strcmp(buf, "1,0")); strcpy(mom_alias, "napali"); - fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3", buf, sizeof(buf)) == PBSE_NONE); + fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3", + buf, + sizeof(buf), + -1) == PBSE_NONE); fail_unless(!strcmp(buf, "1,2,3"), buf); + + fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3", + buf, + sizeof(buf), + 70) == PBSE_NONE); + fail_unless(!strcmp(buf, "0,1,2"), buf); + + fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3", + buf, + sizeof(buf), + 80) == PBSE_NONE); + fail_unless(!strcmp(buf, "0,1,2"), buf); - fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3+waimea-gpu/0+waimea-gpu/1+waimea-gpu/2", buf, sizeof(buf)) == PBSE_NONE); + fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3+waimea-gpu/0+waimea-gpu/1+waimea-gpu/2", + buf, + sizeof(buf), + -1) == PBSE_NONE); fail_unless(!strcmp(buf, "1,2,3"), buf); } END_TEST From 1ea289bcd1e54768eb8a17c45abf5585456acb97 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 18 Nov 2016 15:36:00 -0700 Subject: [PATCH 066/312] TRQ-3854. Fix a crash due to the batch request refactor. --- src/server/req_signal.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/server/req_signal.c b/src/server/req_signal.c index 4ae4a792e2..ee299e0bbb 100644 --- a/src/server/req_signal.c +++ b/src/server/req_signal.c @@ -274,11 +274,16 @@ int issue_signal( if ((rc == PBSE_NONE) && (pjob != NULL)) { - strcpy(jobid, pjob->ji_qs.ji_jobid); - unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); - func(&newreq); + if (func != NULL) + { + strcpy(jobid, pjob->ji_qs.ji_jobid); + + unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); - *pjob_ptr = svr_find_job(jobid, TRUE); + func(&newreq); + + *pjob_ptr = svr_find_job(jobid, TRUE); + } } else if ((extend != NULL) && (!strcmp(extend, RERUNFORCE))) From 76bd50ad597df767e86ef0b0ac40d727a9e4d322 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 22 Nov 2016 16:06:36 -0700 Subject: [PATCH 067/312] TRQ-3858. Fix some valgrind errors. --- src/resmom/start_exec.c | 10 +- src/server/req_delete.c | 140 ++++++++++++++++++++++------ src/server/req_manager.c | 35 +++---- src/server/req_modify.c | 9 +- src/server/req_shutdown.c | 6 +- src/test/req_delete/scaffolding.c | 7 +- src/test/req_delete/test_uut.c | 29 +++--- src/test/req_shutdown/scaffolding.c | 4 + 8 files changed, 160 insertions(+), 80 deletions(-) diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index a4afa8e93a..f41ca6f041 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -1350,9 +1350,9 @@ int TMakeTmpDir( * @param exec_str - the string we're parsing * @param buf - where the list of absolute mic indices goes * @param buf_size - maximum size that can be written into buf - * @param cuda_version - -1 if we're not doing gpus, or the version of cuda otherwise - * If we are doing cuda, and we're above version 7.0, we ignore the index - * in the string and count from 0. + * @param param_cuda_version - -1 if we're not doing gpus, or the version of cuda otherwise + * If we are doing cuda, and we're above version 7.0, we ignore the index + * in the string and count from 0. */ int get_indices_from_exec_str( @@ -1360,7 +1360,7 @@ int get_indices_from_exec_str( const char *exec_str, char *buf, int buf_size, - int cuda_version) + int param_cuda_version) { char *work_str; @@ -1417,7 +1417,7 @@ int get_indices_from_exec_str( { // CUDA versions 7.0 and greater always index from 0 on up for the program, so if you // have gpus 2 and 3, we should pass 0 and 1 - if (cuda_version >= 70) + if (param_cuda_version >= 70) index = relative_count; else index = strtol(slash+1, NULL, 10) + numa_offset; diff --git a/src/server/req_delete.c b/src/server/req_delete.c index e692854354..4285e4a8a1 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -775,17 +775,23 @@ int execute_job_delete( -void *delete_all_work( +/* + * delete_all_work() + * + * Does the work to delete all jobs + * @param preq - the batch_request to delete all jobs. + * @return PBSE_NONE on success or the number of failed deletes. + */ - void *vp) +int delete_all_work( - { - batch_request *preq = (batch_request *)vp; + batch_request *preq) + { if (qdel_all_tracker.start_deleting_all_if_possible(preq->rq_user, preq->rq_perm) == false) { reply_ack(preq); - return(NULL); + return(PBSE_NONE); } batch_request preq_dup(*preq); @@ -852,7 +858,6 @@ void *delete_all_work( { // execute_job_delete() handles mutex so don't unlock on exit job_mutex.set_unlock_on_exit(false); - reply_ack(&preq_dup); } } @@ -872,6 +877,7 @@ void *delete_all_work( if (failed_deletes == 0) { reply_ack(preq); + rc = PBSE_NONE; } else { @@ -880,27 +886,62 @@ void *delete_all_work( total_jobs); req_reject(PBSE_SYSTEM, 0, preq, NULL, tmpLine); + rc = failed_deletes; } - return(NULL); + return(rc); } /* END delete_all_work() */ +/* + * delete_all_task() + * + * The asynchronous work task for deleting all jobs + * @param vp - the batch_request to delete all jobs. Always deleted by the end of this function, + * unless it is NULL. + */ + +void *delete_all_task( + + void *vp) + + { + batch_request *preq = (batch_request *)vp; + + if (preq != NULL) + { + delete_all_work(preq); + + delete preq; + } + + return(NULL); + } // END delete_all_task() + + + +/* + * handle_delete_all() + * + * Handles the logic for deleting all jobs + * @param preq - tells us what to delete and how. + * @param Msg - (optional) a message explaining why we're deleting these jobs + * @return PBSE_NONE + */ int handle_delete_all( batch_request *preq, - batch_request *preq_tmp, char *Msg) { - /* preq_tmp is not null if this is an asynchronous request */ - if (preq_tmp != NULL) + if ((preq->rq_extend != NULL) && + (!strncmp(preq->rq_extend,delasyncstr,strlen(delasyncstr)))) { - reply_ack(preq_tmp); + reply_ack(preq); preq->rq_noreply = true; /* set for no more replies */ - enqueue_threadpool_request(delete_all_work, preq, request_pool); + enqueue_threadpool_request(delete_all_task, new batch_request(*preq), request_pool); } else delete_all_work(preq); @@ -910,13 +951,20 @@ int handle_delete_all( -void *single_delete_work( +/* + * single_delete_work() + * + * Performs the work of deleting a single job + * @param preq - the batch_request instructing us what to delete and how + * @return PBSE_* to indicate status + */ - void *vp) +int single_delete_work( + + batch_request *preq) { int rc = -1; - batch_request *preq = (batch_request *)vp; char *jobid = preq->rq_ind.rq_delete.rq_objname; job *pjob; char *Msg = preq->rq_extend; @@ -927,7 +975,8 @@ void *single_delete_work( if (pjob == NULL) { - req_reject(PBSE_JOBNOTFOUND, 0, preq, NULL, "job unexpectedly deleted"); + rc = PBSE_JOBNOTFOUND; + req_reject(rc, 0, preq, NULL, "job unexpectedly deleted"); } else { @@ -937,17 +986,57 @@ void *single_delete_work( if ((rc == PBSE_NONE) || (rc == PURGE_SUCCESS)) + { reply_ack(preq); + rc = PBSE_NONE; + } } - return(NULL); + return(rc); } /* END single_delete_work() */ + +/* + * single_delete_task() + * + * The taskpool task for performing a single job delete + * + * @param vp - a void pointer to the batch_request pointer we need. This is always deleted + * by the end of this function, unless it is NULL. + */ + +void *single_delete_task( + + void *vp) + + { + if (vp != NULL) + { + batch_request *preq = (batch_request *)vp; + + single_delete_work(preq); + + delete preq; + } + + return(NULL); + } // END single_delete_task() + + + +/* + * handle_single_delete() + * + * Handles the logic for deleting a single a job. + * @param preq - the batch_request telling us what to delete and how. + * @param Msg - (optional) a message stating why we're deleting this job. + * @return PBSE_NONE + */ + int handle_single_delete( batch_request *preq, - batch_request *preq_tmp, char *Msg) { @@ -965,11 +1054,12 @@ int handle_single_delete( unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); /* send the asynchronous reply if needed */ - if (preq_tmp != NULL) + if ((preq->rq_extend != NULL) && + (!strncmp(preq->rq_extend, delasyncstr, strlen(delasyncstr)))) { - reply_ack(preq_tmp); + reply_ack(preq); preq->rq_noreply = true; /* set for no more replies */ - enqueue_threadpool_request(single_delete_work, preq, async_pool); + enqueue_threadpool_request(single_delete_task, new batch_request(*preq), async_pool); } else single_delete_work(preq); @@ -980,7 +1070,6 @@ int handle_single_delete( - /* * req_deletejob - service the Delete Job Request * @@ -1028,7 +1117,6 @@ int req_deletejob( { char *Msg = NULL; - struct batch_request *preq_tmp = NULL; char log_buf[LOCAL_LOG_BUF_SIZE]; /* check if we are getting a purgecomplete from scheduler */ @@ -1068,18 +1156,16 @@ int req_deletejob( */ snprintf(log_buf,sizeof(log_buf), "Deleting job asynchronously"); log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,preq->rq_ind.rq_delete.rq_objname,log_buf); - - preq_tmp = new batch_request(*preq); } } if (strcasecmp(preq->rq_ind.rq_delete.rq_objname,"all") == 0) { - handle_delete_all(preq, preq_tmp, Msg); + handle_delete_all(preq, Msg); } else { - handle_single_delete(preq, preq_tmp, Msg); + handle_single_delete(preq, Msg); } return(PBSE_NONE); diff --git a/src/server/req_manager.c b/src/server/req_manager.c index 8c5c8b28ce..ffaf486726 100644 --- a/src/server/req_manager.c +++ b/src/server/req_manager.c @@ -2027,29 +2027,25 @@ static bool requeue_or_delete_jobs( if(pjob != NULL) { char *dup_jobid = strdup(pjob->ji_qs.ji_jobid); - batch_request *brRerun = new batch_request(PBS_BATCH_Rerun); - batch_request *brDelete = new batch_request(PBS_BATCH_DeleteJob); - if((brRerun == NULL)||(brDelete == NULL)) - { - free_br(brRerun); - free_br(brDelete); - free(dup_jobid); - return true; //Ignoring this error. - } - strcpy(brRerun->rq_ind.rq_rerun,pjob->ji_qs.ji_jobid); - strcpy(brDelete->rq_ind.rq_delete.rq_objname,pjob->ji_qs.ji_jobid); - brRerun->rq_conn = PBS_LOCAL_CONNECTION; - brDelete->rq_conn = PBS_LOCAL_CONNECTION; - brRerun->rq_perm = preq->rq_perm; - brDelete->rq_perm = preq->rq_perm; - brDelete->rq_ind.rq_delete.rq_objtype = MGR_OBJ_JOB; - brDelete->rq_ind.rq_delete.rq_cmd = MGR_CMD_DELETE; + batch_request brRerun(PBS_BATCH_Rerun); + batch_request brDelete(PBS_BATCH_DeleteJob); + + strcpy(brRerun.rq_ind.rq_rerun,pjob->ji_qs.ji_jobid); + strcpy(brDelete.rq_ind.rq_delete.rq_objname,pjob->ji_qs.ji_jobid); + + brRerun.rq_conn = PBS_LOCAL_CONNECTION; + brDelete.rq_conn = PBS_LOCAL_CONNECTION; + brRerun.rq_perm = preq->rq_perm; + brDelete.rq_perm = preq->rq_perm; + brDelete.rq_ind.rq_delete.rq_objtype = MGR_OBJ_JOB; + brDelete.rq_ind.rq_delete.rq_cmd = MGR_CMD_DELETE; + unlock_ji_mutex(pjob,__func__,NULL,LOGLEVEL); pnode->tmp_unlock_node(__func__, NULL, LOGLEVEL); - int rc = req_rerunjob(brRerun); + int rc = req_rerunjob(&brRerun); if(rc != PBSE_NONE) { - rc = req_deletejob(brDelete); + rc = req_deletejob(&brDelete); if(rc == PBSE_NONE) { get_svr_attr_l(SRV_ATR_TimeoutForJobDelete, &delete_timeout); @@ -2064,7 +2060,6 @@ static bool requeue_or_delete_jobs( } else { - free_br(brDelete); get_svr_attr_l(SRV_ATR_TimeoutForJobRequeue, &requeue_timeout); if(!wait_for_job_state(*jid,JOB_STATE_QUEUED,requeue_timeout)) diff --git a/src/server/req_modify.c b/src/server/req_modify.c index b563874720..2e5337e1ad 100644 --- a/src/server/req_modify.c +++ b/src/server/req_modify.c @@ -653,7 +653,6 @@ int modify_job( else { reply_ack(preq); - preq = NULL; } if (copy_checkpoint_files) @@ -730,12 +729,12 @@ int modify_whole_array( else { /* NO_MOM_RELAY will prevent modify_job from calling relay_to_mom */ - batch_request *array_req = new batch_request(*preq); - array_req->update_object_id(i); + batch_request array_req(*preq); + array_req.update_object_id(i); mutex_mgr job_mutex(pjob->ji_mutex, true); pthread_mutex_unlock(pa->ai_mutex); - array_req->rq_noreply = true; - rc = modify_job((void **)&pjob, plist, array_req, checkpoint_req, NO_MOM_RELAY); + array_req.rq_noreply = true; + rc = modify_job((void **)&pjob, plist, &array_req, checkpoint_req, NO_MOM_RELAY); if (rc != PBSE_NONE) { modify_job_rc = rc; diff --git a/src/server/req_shutdown.c b/src/server/req_shutdown.c index 20395dd29b..90afc11e82 100644 --- a/src/server/req_shutdown.c +++ b/src/server/req_shutdown.c @@ -114,7 +114,7 @@ /* Private Data Items */ -static struct batch_request *pshutdown_request = 0; +static batch_request *pshutdown_request = NULL; /* Global Data Items: */ @@ -250,6 +250,8 @@ void shutdown_ack(void) { reply_ack(pshutdown_request); + delete pshutdown_request; + pshutdown_request = 0; } @@ -288,7 +290,7 @@ void req_shutdown( msg_daemonname, log_buf); - pshutdown_request = preq; /* save for reply from main() when done */ + pshutdown_request = new batch_request(*preq); /* save for reply from main() when done */ svr_shutdown(preq->rq_ind.rq_shutdown); diff --git a/src/test/req_delete/scaffolding.c b/src/test/req_delete/scaffolding.c index db6d2ab44d..b5cf59b128 100644 --- a/src/test/req_delete/scaffolding.c +++ b/src/test/req_delete/scaffolding.c @@ -53,14 +53,11 @@ job_array *get_jobs_array(job **pjob) void account_record(int acctype, job *pjob, const char *text) { - fprintf(stderr, "The call to acctype needs to be mocked!!\n"); - exit(1); } int job_save(job *pjob, int updatetype, int mom_port) { - fprintf(stderr, "The call to job_save needs to be mocked!!\n"); - exit(1); + return(0); } int svr_job_purge(job *pjob, int leaveSpoolFiles) @@ -76,8 +73,6 @@ void chk_job_req_permissions(job **pjob_ptr, struct batch_request *preq) void svr_mailowner(job *pjob, int mailpoint, int force, const char *text) { - fprintf(stderr, "The call to svr_mailowner needs to be mocked!!\n"); - exit(1); } long attr_ifelse_long(pbs_attribute *attr1, pbs_attribute *attr2, long deflong) diff --git a/src/test/req_delete/test_uut.c b/src/test/req_delete/test_uut.c index 8842037da0..907b1a664b 100644 --- a/src/test/req_delete/test_uut.c +++ b/src/test/req_delete/test_uut.c @@ -21,15 +21,15 @@ int delete_inactive_job(job **, const char *); void force_purge_work(job *pjob); -void *delete_all_work(void *vp); +int delete_all_work(batch_request *); void ensure_deleted(struct work_task *ptask); int apply_job_delete_nanny(job *pjob, int delay); void job_delete_nanny(struct work_task *ptask); void post_job_delete_nanny(batch_request *preq_sig); int forced_jobpurge(job *pjob, batch_request *preq); void post_delete_mom2(struct work_task *pwt); -int handle_delete_all(batch_request *preq, batch_request *preq_tmp, char *Msg); -int handle_single_delete(batch_request *preq, batch_request *preq_tmp, char *Msg); +int handle_delete_all(batch_request *preq, char *Msg); +int handle_single_delete(batch_request *preq, char *Msg); bool exit_called; extern int depend_term_called; extern long keep_seconds; @@ -79,21 +79,24 @@ START_TEST(test_handle_single_delete) { batch_request *preq = new batch_request(); strcpy(preq->rq_ind.rq_delete.rq_objname, "2.napali"); - fail_unless(handle_single_delete(preq, preq, NULL) == PBSE_NONE); + fail_unless(handle_single_delete(preq, NULL) == PBSE_NONE); fail_unless(preq->rq_noreply == FALSE); + // Now an asynchronous delete + preq->rq_extend = strdup(DELASYNC); strcpy(preq->rq_ind.rq_delete.rq_objname, "1.napali"); - fail_unless(handle_single_delete(preq, preq, NULL) == PBSE_NONE); - fail_unless(preq->rq_noreply == TRUE); + fail_unless(handle_single_delete(preq, NULL) == PBSE_NONE); + fail_unless(preq->rq_noreply == true); } END_TEST START_TEST(test_handle_delete_all) { batch_request preq; + preq.rq_extend = strdup(DELASYNC); - fail_unless(handle_delete_all(&preq, &preq, NULL) == PBSE_NONE); - fail_unless(preq.rq_noreply == TRUE); + fail_unless(handle_delete_all(&preq, NULL) == PBSE_NONE); + fail_unless(preq.rq_noreply == true); } END_TEST @@ -143,12 +146,8 @@ END_TEST START_TEST(test_delete_all_work) { //struct all_jobs alljobs; - job *pjob; - batch_request *preq; - - pjob = new job(); - - preq = new batch_request(); + job *pjob = new job(); + batch_request *preq = new batch_request(); preq->rq_extend = strdup(delpurgestr); nanny = 0; @@ -156,7 +155,7 @@ START_TEST(test_delete_all_work) // no lock should remain on job after delete_all_work() // so test by making sure we can set lock - fail_unless(delete_all_work((void *) preq) == NULL); + fail_unless(delete_all_work(preq) == PBSE_NONE); fail_unless(pthread_mutex_trylock(pjob->ji_mutex) == 0); nanny = 1; diff --git a/src/test/req_shutdown/scaffolding.c b/src/test/req_shutdown/scaffolding.c index 9b42f3353d..30056aa97f 100644 --- a/src/test/req_shutdown/scaffolding.c +++ b/src/test/req_shutdown/scaffolding.c @@ -158,3 +158,7 @@ AvlTree AVL_clear_tree(AvlTree a) fprintf(stderr, "The call to AVL_clear_tree needs to be mocked!!\n"); exit(1); } + +batch_request::batch_request(const batch_request &other) {} +batch_request::~batch_request() {} + From 02aa1bf7b21a6d7040323157670d44a153cc3ac4 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 22 Nov 2016 16:27:32 -0700 Subject: [PATCH 068/312] TRQ-3858. Scaffolding update. --- src/test/req_manager/scaffolding.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/req_manager/scaffolding.c b/src/test/req_manager/scaffolding.c index 2859d50708..1cd12c1ead 100644 --- a/src/test/req_manager/scaffolding.c +++ b/src/test/req_manager/scaffolding.c @@ -516,3 +516,4 @@ batch_request::batch_request(int type) : rq_type(type) { } +batch_request::~batch_request() {} From 75b58d25ed192accccc1d20c9c4ae3983cddabca Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 28 Nov 2016 15:50:47 -0700 Subject: [PATCH 069/312] TRQ-3870. Fix an invalid read when dependencies have been satisfied. --- src/server/req_register.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/server/req_register.c b/src/server/req_register.c index 35711f7091..a3f08966e0 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -1580,9 +1580,9 @@ int depend_on_exec( job *pjob) { - struct depend *pdep; - depend_job *pdj; - char jobid[PBS_MAXSVRJOBID+1]; + depend *pdep; + depend_job *pdj; + char jobid[PBS_MAXSVRJOBID+1]; strcpy(jobid, pjob->ji_qs.ji_jobid); /* If any jobs come after my start, release them */ @@ -1592,10 +1592,11 @@ int depend_on_exec( if (pdep != NULL) { + std::vector depend_jobs = pdep->dp_jobs; - for (unsigned int i = 0; i < pdep->dp_jobs.size(); i++) + for (unsigned int i = 0; i < depend_jobs.size(); i++) { - pdj = pdep->dp_jobs[i]; + pdj = depend_jobs[i]; if (send_depend_req(pjob, pdj, @@ -1658,9 +1659,6 @@ int depend_on_exec( - - - /* depend_on_term - Perform actions if job has "afterany, afterok, afternotok" * dependencies, send "register-release" or register-delete" as * appropriate. From 6afb74cd1263b19d2eb25e833c37380bdda41291 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 28 Nov 2016 17:38:11 -0700 Subject: [PATCH 070/312] TRQ-3856. Pass the value of -C to the submit filter as well. --- src/cmds/qsub_functions.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index ea8981a669..2369bbe953 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -1198,6 +1198,8 @@ static int get_script( directive_prefix[0] = '\\'; strcat(directive_prefix, ArgV[index]); directive_prefix_on = false; + + cfilter += directive_prefix; continue; } From c2dabfe658068f5992a88ae2d20ee8a7ae3f2e71 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 28 Nov 2016 17:38:33 -0700 Subject: [PATCH 071/312] TRQ-3869. Fix some valgrind issues. --- src/resmom/nvidia.c | 2 +- src/resmom/parse_config.c | 4 ++++ src/server/array_func.c | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index c03f21b3f8..8e420dde96 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -2235,7 +2235,7 @@ bool matches_version_string( int tmp_version; bool match = false; - tmp_version += strtol(ptr, &end, 10) * 10; + tmp_version = strtol(ptr, &end, 10) * 10; if (*end == '.') { diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index d2920cbc3a..1a05e3df3c 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -2293,7 +2293,11 @@ void reset_config_vars() memset(MOMConfigVersion, 0, sizeof(MOMConfigVersion)); MOMConfigDownOnError = 0; MOMConfigRestart = 0; +#ifdef PENABLE_LINUX_CGROUPS + MOMCudaVisibleDevices = 0; +#else MOMCudaVisibleDevices = 1; +#endif wallfactor = 1.00; pcphosts.clear(); pe_alarm_time = PBS_PROLOG_TIME; diff --git a/src/server/array_func.c b/src/server/array_func.c index f794048f1c..0677864860 100644 --- a/src/server/array_func.c +++ b/src/server/array_func.c @@ -1800,12 +1800,12 @@ int modify_array_range( } else { - batch_request *array_req = new batch_request(*preq); - array_req->update_object_id(index); + batch_request array_req(*preq); + array_req.update_object_id(index); mutex_mgr pjob_mutex = mutex_mgr(pjob->ji_mutex, true); pthread_mutex_unlock(pa->ai_mutex); - array_req->rq_noreply = true; - modify_job((void **)&pjob, plist, array_req, checkpoint_req, NO_MOM_RELAY); + array_req.rq_noreply = true; + modify_job((void **)&pjob, plist, &array_req, checkpoint_req, NO_MOM_RELAY); pa = get_jobs_array(&pjob); if (pa == NULL) From 3d71a3b2b2861ccaae1da0ce9dfb5d800ae7ae07 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 29 Nov 2016 11:42:20 -0700 Subject: [PATCH 072/312] DOC-2819. Update a comment about a server parameter. --- src/include/qmgr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/qmgr.h b/src/include/qmgr.h index 595139fccd..fc312fa2ad 100644 --- a/src/include/qmgr.h +++ b/src/include/qmgr.h @@ -306,7 +306,7 @@ void disconnect_from_server(); "resources_cost - the cost factors of resources. Used for sync. job starting\n" \ "resources_default - the default resource value when the job does not specify\n" \ "resource_max - the maximum amount of resources that are on the system\n" \ - "scheduler_iteration - the amount of seconds between timed scheduler iterations\n" \ + "scheduler_iteration - has no effect\n" \ "scheduling - when true the server should tell the scheduler to run\n" \ "system_cost - arbitrary value factored into resource costs\n" \ "use_jobs_subdirs - when true divide storage of jobs into subdirectories in $PBS_HOME/server_priv/{jobs,arrays}\n" \ From 789710b519314f4153b3fb354b5ca0c32af90dca Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 29 Nov 2016 16:45:04 -0700 Subject: [PATCH 073/312] Fix a scaffolding error. --- src/test/array_func/scaffolding.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/array_func/scaffolding.c b/src/test/array_func/scaffolding.c index 9e667cffd1..e06c06f6d4 100644 --- a/src/test/array_func/scaffolding.c +++ b/src/test/array_func/scaffolding.c @@ -386,3 +386,4 @@ batch_request::batch_request(const batch_request &other) void batch_request::update_object_id(int job_index) {} +batch_request::~batch_request() {} From 8de0d8f99ab743353c902243c8e6673a71c292bf Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 30 Nov 2016 23:34:50 -0700 Subject: [PATCH 074/312] TRQ-3861. Fix an error with detecting GPUs and their locality. This is for the cgroup and enforcement code. --- src/include/machine.hpp | 7 +- src/lib/Libutils/machine.cpp | 10 +++ src/lib/Libutils/numa_chip.cpp | 5 ++ src/lib/Libutils/numa_pci_device.cpp | 5 +- src/lib/Libutils/numa_socket.cpp | 39 ++++++++--- src/resmom/accelerators_numa.cpp | 100 +++++++++++++++++++++------ 6 files changed, 134 insertions(+), 32 deletions(-) diff --git a/src/include/machine.hpp b/src/include/machine.hpp index 13b293d688..14e386e880 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -2,6 +2,7 @@ #define __MACHINE_HPP__ #include #include +#include #include #include #ifdef NVIDIA_GPUS @@ -147,6 +148,7 @@ class Chip int get_id() const; int getTotalCores() const; int getTotalThreads() const; + int get_total_gpus() const; int getAvailableCores() const; int getAvailableThreads() const; hwloc_uint64_t getAvailableMemory() const; @@ -254,6 +256,8 @@ class Socket int getTotalChips() const; int getTotalCores() const; int getTotalThreads() const; + int get_total_gpus() const; + int get_available_gpus() const; int getAvailableChips() const; int getAvailableCores() const; int get_free_cores() const; @@ -307,7 +311,7 @@ class Machine vector allocations; #ifdef NVIDIA_GPUS #ifdef NVML_API - hwloc_obj_t get_non_nvml_device(hwloc_topology_t topology, nvmlDevice_t device); + hwloc_obj_t get_non_nvml_device(hwloc_topology_t topology, nvmlDevice_t device, std::set &); #endif #endif @@ -328,6 +332,7 @@ class Machine int getTotalChips() const; int getTotalCores() const; int getTotalThreads() const; + int get_total_gpus() const; int getAvailableSockets() const; int getAvailableChips() const; int getAvailableMemory() const; diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index 1baf0a636d..05985d5d87 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -431,6 +431,16 @@ int Machine::getTotalSockets() const return(this->totalSockets); } +int Machine::get_total_gpus() const + { + int total_gpus = 0; + + for (size_t i = 0; i < this->sockets.size(); i++) + total_gpus += this->sockets[i].get_total_gpus(); + + return(total_gpus); + } + int Machine::getTotalChips() const { return(this->totalChips); diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index 32c8dc199f..41d21c342c 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -574,6 +574,11 @@ int Chip::getTotalThreads() const return(this->totalThreads); } +int Chip::get_total_gpus() const + { + return(this->total_gpus); + } + int Chip::getAvailableCores() const { if (this->chip_exclusive == true) diff --git a/src/lib/Libutils/numa_pci_device.cpp b/src/lib/Libutils/numa_pci_device.cpp index b6a8f9dc30..76e0e2eccd 100644 --- a/src/lib/Libutils/numa_pci_device.cpp +++ b/src/lib/Libutils/numa_pci_device.cpp @@ -93,7 +93,10 @@ void PCI_Device::displayAsString( stringstream &out) const { - out << " pci " << this->id << " " << this->name << "\n"; + if (this->type == GPU) + out << " GPU " << this->id << " " << this->name << "\n"; + else + out << " MIC " << this->id << " " << this->name << "\n"; } // end displayasstring() void PCI_Device::setName( diff --git a/src/lib/Libutils/numa_socket.cpp b/src/lib/Libutils/numa_socket.cpp index 6162fa12aa..492d56213b 100644 --- a/src/lib/Libutils/numa_socket.cpp +++ b/src/lib/Libutils/numa_socket.cpp @@ -311,6 +311,26 @@ int Socket::getTotalChips() const return(this->chips.size()); } +int Socket::get_total_gpus() const + { + int total_gpus = 0; + + for (size_t i = 0; i < this->chips.size(); i++) + total_gpus += this->chips[i].get_total_gpus(); + + return(total_gpus); + } + +int Socket::get_available_gpus() const + { + int available_gpus; + + for (size_t i = 0; i < this->chips.size(); i++) + available_gpus += this->chips[i].get_available_gpus(); + + return(available_gpus); + } + int Socket::getAvailableChips() const { int available_numa_nodes = 0; @@ -834,17 +854,20 @@ bool Socket::fits_on_socket( if (remaining.place_cpus > 0) max_cpus = remaining.place_cpus; - if ((remaining.cores_only == true) && - (this->get_free_cores() >= max_cpus)) - fits = true; - else if (remaining.place_type == exclusive_legacy) + if (this->get_available_gpus() >= remaining.gpus) { - if (this->get_free_cores() >= max_cpus) + if ((remaining.cores_only == true) && + (this->get_free_cores() >= max_cpus)) + fits = true; + else if (remaining.place_type == exclusive_legacy) + { + if (this->get_free_cores() >= max_cpus) + fits = true; + } + else if ((remaining.cores_only == false) && + (this->getAvailableThreads() >= max_cpus)) fits = true; } - else if ((remaining.cores_only == false) && - (this->getAvailableThreads() >= max_cpus)) - fits = true; } return(fits); diff --git a/src/resmom/accelerators_numa.cpp b/src/resmom/accelerators_numa.cpp index 44b314e8b5..ddf9230ed3 100644 --- a/src/resmom/accelerators_numa.cpp +++ b/src/resmom/accelerators_numa.cpp @@ -21,7 +21,25 @@ void log_nvml_error(nvmlReturn_t rc, char* gpuid, const char* id); #ifdef NVML_API -hwloc_obj_t Machine::get_non_nvml_device(hwloc_topology_t topology, nvmlDevice_t device) + +/* + * get_non_nvml_device() + * + * hwloc's method to identify a GPU failed, so use ours. Theirs fails a lot. + * + * @param topology - the topology that the device should fall under + * @param device - the nvml device handle + * @param identified - a list of the previously identified devices to make sure + * we aren't storing the same one multiple times. + * @return an hwloc obj pointing to the GPU + */ + +hwloc_obj_t Machine::get_non_nvml_device( + + hwloc_topology_t topology, + nvmlDevice_t device, + std::set &identified) + { hwloc_obj_t osdev; nvmlReturn_t nvres; @@ -33,27 +51,39 @@ hwloc_obj_t Machine::get_non_nvml_device(hwloc_topology_t topology, nvmlDevice_t return NULL; } - nvres = nvmlDeviceGetPciInfo(device, &pci); - if (NVML_SUCCESS != nvres) - return NULL; + nvres = nvmlDeviceGetPciInfo(device, &pci); + if (NVML_SUCCESS != nvres) + return(NULL); - osdev = NULL; - while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) - { - hwloc_obj_t pcidev = osdev->parent; - if (strncmp(osdev->name, "card", 4)) - continue; - if (pcidev - && pcidev->type == HWLOC_OBJ_PCI_DEVICE - && pcidev->attr->pcidev.domain == pci.domain - && pcidev->attr->pcidev.bus == pci.bus - && pcidev->attr->pcidev.dev == pci.device - && pcidev->attr->pcidev.func == 0) - return osdev; - } + osdev = NULL; + while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) + { + hwloc_obj_t pcidev = osdev->parent; + if ((strncmp(osdev->name, "card", 4)) && + (strncmp(osdev->name, "controlD", 8))) + continue; + + if ((pcidev != NULL) && + (pcidev->type == HWLOC_OBJ_PCI_DEVICE) && + (pcidev->attr->pcidev.domain == pci.domain) && + (pcidev->attr->pcidev.bus == pci.bus) && + (pcidev->attr->pcidev.dev == pci.device) && + (pcidev->attr->pcidev.func == 0)) + return(osdev); + } + + // We haven't found it yet, but there are GPUs and hwloc knows it. Just match on type. + osdev = NULL; + while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) + { + if ((osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_GPU) && + (identified.find(osdev) == identified.end())) + return(osdev); + } return(NULL); - } + } // END get_non_nvml_device() + int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t topology) @@ -74,12 +104,14 @@ int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t t } unsigned int device_count = 0; + unsigned int found_devices = 0; /* Get the device count. */ rc = nvmlDeviceGetCount(&device_count); if (rc == NVML_SUCCESS) { nvmlDevice_t gpu; + std::set identified; /* Get the nvml device handle at each index */ for (unsigned int idx = 0; idx < device_count; idx++) @@ -100,23 +132,47 @@ int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t t gpu_obj = hwloc_nvml_get_device_osdev(topology, gpu); if (gpu_obj == NULL) { - /* This was not an nvml device. We will look for a "card" device (GeForce or Quadra) */ - gpu_obj = this->get_non_nvml_device(topology, gpu); + // This was not an nvml device. We'll look for a different GPU - many aren't recognized, + // including k80s. + gpu_obj = this->get_non_nvml_device(topology, gpu, identified); if (gpu_obj == NULL) - continue; + continue; } + + identified.insert(gpu_obj); + found_devices++; /* The ancestor was not a numa chip. Is it the machine? */ ancestor_obj = hwloc_get_ancestor_obj_by_type(topology, HWLOC_OBJ_MACHINE, gpu_obj); if (ancestor_obj != NULL) { + char buf[MAXLINE]; + rc = nvmlDeviceGetName(gpu, buf, sizeof(buf)); + PCI_Device new_device; new_device.initializePCIDevice(gpu_obj, idx, topology); store_device_on_appropriate_chip(new_device); + + // hwloc sees the K80 as a single device. We want to display two to stay in sync + // with NVML, so store an extra for each K80 + if (!strncmp(buf, "Tesla K80", 9)) + { + new_device.setId(new_device.get_id() + 1); + store_device_on_appropriate_chip(new_device); + found_devices++; + idx++; + } } } + + if (found_devices != device_count) + { + sprintf(log_buffer, "NVML reports %u devices, but we only found %u", + device_count, found_devices); + log_err(-1, __func__, log_buffer); + } } else { From 41135d5e2da76c8fefb86d4e801964bd7ae02ae5 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 1 Dec 2016 09:53:57 -0700 Subject: [PATCH 075/312] TRQ-3792 - fix stdout delivery when --disable-spool set --- src/include/mom_func.h | 2 +- src/resmom/prolog.c | 2 +- src/resmom/requests.c | 2 +- src/resmom/start_exec.c | 223 ++++++++++++++++++++++++------ src/test/prolog/scaffolding.c | 3 +- src/test/requests/scaffolding.c | 2 +- src/test/start_exec/scaffolding.c | 5 +- src/test/start_exec/test_uut.c | 150 ++++++++++++++++++++ 8 files changed, 339 insertions(+), 50 deletions(-) diff --git a/src/include/mom_func.h b/src/include/mom_func.h index 6d3c8f09a0..6b08c0ea89 100644 --- a/src/include/mom_func.h +++ b/src/include/mom_func.h @@ -268,7 +268,7 @@ void clear_servers(); void set_jobs_substate(job *pjob, int new_substate); -int become_the_user(job *pjob); +int become_the_user(job *pjob, bool want_effective); bool am_i_mother_superior(const job &pjob); diff --git a/src/resmom/prolog.c b/src/resmom/prolog.c index 63a7424477..6afc41d275 100644 --- a/src/resmom/prolog.c +++ b/src/resmom/prolog.c @@ -1183,7 +1183,7 @@ void prepare_and_run_pelog_as_child( setuid_ext(pbsuser, TRUE); setegid(pbsgroup); - if (become_the_user(pjob) != PBSE_NONE) + if (become_the_user(pjob, false) != PBSE_NONE) { exit(-1); } diff --git a/src/resmom/requests.c b/src/resmom/requests.c index a5a034d648..92de85a647 100644 --- a/src/resmom/requests.c +++ b/src/resmom/requests.c @@ -1271,7 +1271,7 @@ int message_job( } /* only the child reaches here, become the user for root-squashing as well */ - if (become_the_user(pjob) != PBSE_NONE) + if (become_the_user(pjob, false) != PBSE_NONE) { /* log_buffer is populated by become_the_user */ log_err(errno, __func__, log_buffer); diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index f41ca6f041..c7723771a2 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -686,12 +686,14 @@ void exec_bail( * becomes the user for pjob * * @param pjob - the job whose user we should become + * @param want_effective - whether or not to set euid/eguid rather than uid/gid * @return PBSE_BADUSER on failure */ int become_the_user( - job *pjob) + job *pjob, + bool want_effective) { log_buffer[0] = '\0'; @@ -704,20 +706,41 @@ int become_the_user( (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, strerror(errno)); } - else if (setgid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != PBSE_NONE) + else if (want_effective) { - snprintf(log_buffer,sizeof(log_buffer), - "PBS: setgid to %lu for UID = %lu failed: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid, - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); + if (setegid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != PBSE_NONE) + { + snprintf(log_buffer,sizeof(log_buffer), + "PBS: setegid to %lu for UID = %lu failed: %s\n", + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid, + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, + strerror(errno)); + } + else if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, TRUE) < 0) + { + snprintf(log_buffer,sizeof(log_buffer), + "PBS: seteuid to %lu failed: %s\n", + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, + strerror(errno)); + } } - else if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, FALSE) < 0) + else { - snprintf(log_buffer,sizeof(log_buffer), - "PBS: setuid to %lu failed: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); + if (setgid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != PBSE_NONE) + { + snprintf(log_buffer,sizeof(log_buffer), + "PBS: setgid to %lu for UID = %lu failed: %s\n", + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid, + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, + strerror(errno)); + } + else if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, FALSE) < 0) + { + snprintf(log_buffer,sizeof(log_buffer), + "PBS: setuid to %lu failed: %s\n", + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, + strerror(errno)); + } } if (log_buffer[0] != '\0') @@ -737,7 +760,7 @@ int become_the_user_sjr( struct startjob_rtn *sjr) { - if (become_the_user(pjob) != PBSE_NONE) + if (become_the_user(pjob, false) != PBSE_NONE) { if (write_ac_socket(2, log_buffer, strlen(log_buffer)) == -1) { @@ -7800,6 +7823,90 @@ int remove_leading_hostname( } /* END remove_leading_hostname() */ +#if NO_SPOOL_OUTPUT == 1 +/* + * save_supplementary_group_list + * + * saves the supplementary group list for calling process + * into a buffer + * + * see restore_supplementary_group_list + * + * Note: it is up to the caller to free the allocated space + * malloc'ed in this function. This is normally done in + * restore_supplementary_group_list. + * + * @param *grp_list_size - pointer to size of group list + * @param **grp_list - pointer to address of group list + * @return 0 if group list saved to buffer, -1 otherwise + */ + +int save_supplementary_group_list( + + int *grp_list_size, /* O */ + gid_t **grp_list) /* O */ + + { + int ngroups_expected; + + // get expected number of groups + ngroups_expected = getgroups(0, NULL); + if (ngroups_expected < 0) + return(-1); + + // allocate space for group list + *grp_list = (gid_t *)malloc(ngroups_expected * sizeof(gid_t)); + if (*grp_list == NULL) + return(-1); + + // get the group list + *grp_list_size = getgroups(ngroups_expected, *grp_list); + if (*grp_list_size != ngroups_expected) + { + free(*grp_list); + *grp_list = NULL; + return(-1); + } + + // success + return(0); + } + + +/* + * restore_supplementary_group_list + * + * restores the supplementary group list for calling process + * from a buffer and frees the buffer space + * + * see save_supplementary_group_list + * + * @param grp_list_size - size of group list + * @param *grp_list - pointer to saved group list + * @return 0 if group list restored from buffer, -1 otherwise + */ + +int restore_supplementary_group_list( + + int grp_list_size, /* I */ + gid_t *grp_list) /* I */ + + { + int rc = -1; + + if (grp_list == NULL) + return(rc); + + // restore the supplementary group list + if (setgroups((size_t)grp_list_size, grp_list) == 0) + rc = 0; + + // free the buffer space + free(grp_list); + + return(rc); + } +#endif // NO_SPOOL_OUTPUT /* @@ -8036,6 +8143,9 @@ char *std_file_name( if (spoolasfinalname == FALSE) { + gid_t *grp_list_saved; + int grp_list_saved_size; + /* force all output to user's HOME */ snprintf(path, sizeof(path), "%s", pjob->ji_grpcache->gc_homedir); @@ -8043,14 +8153,67 @@ char *std_file_name( /* if it's not a directory, just use $HOME us usual */ snprintf(path_alt, sizeof(path_alt), "%s/.pbs_spool/", path); - if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, TRUE) == -1) + // For network file systems like NFS and GPFS and regardless of the root + // squashing mode (enabled or disabled) the user's supplementary group + // list must be set for the caller before doing the stat() call below. + + // save supplementary groups + if (save_supplementary_group_list(&grp_list_saved_size, &grp_list_saved) < 0) { + snprintf(log_buffer,sizeof(log_buffer), + "failed to save supplementary groups"); + log_err(errno, __func__, log_buffer); + return(NULL); } + // set effective uid, gid + if (become_the_user(pjob, true) != PBSE_NONE) + { + restore_supplementary_group_list(grp_list_saved_size, grp_list_saved); + setegid(pbsgroup); + + log_err(errno, __func__, log_buffer); + return(NULL); + } + + // now do the stat() rcstat = stat(path_alt, &myspooldir); - setuid_ext(pbsuser, TRUE); + // restore the euid + if (setuid_ext(pbsuser, TRUE) == -1) + { + restore_supplementary_group_list(grp_list_saved_size, grp_list_saved); + setegid(pbsgroup); + + snprintf(log_buffer,sizeof(log_buffer), + "failed to restore the euid"); + log_err(errno, __func__, log_buffer); + + return(NULL); + } + + // restore supplementary groups + if (restore_supplementary_group_list(grp_list_saved_size, grp_list_saved) < 0) + { + setegid(pbsgroup); + + snprintf(log_buffer,sizeof(log_buffer), + "failed to restore supplementary groups"); + log_err(errno, __func__, log_buffer); + + return(NULL); + } + + // restore the egid + if (setegid(pbsgroup) != 0) + { + snprintf(log_buffer,sizeof(log_buffer), + "failed to restore the egid"); + log_err(errno, __func__, log_buffer); + + return(NULL); + } if ((rcstat == 0) && (S_ISDIR(myspooldir.st_mode))) snprintf(path, sizeof(path), "%s", path_alt); @@ -8220,36 +8383,8 @@ int open_std_file( (geteuid() != pjob->ji_qs.ji_un.ji_momt.ji_exuid)) #endif { - if (setgroups(pjob->ji_grpcache->gc_ngroup,(gid_t *)pjob->ji_grpcache->gc_groups) != PBSE_NONE) - { - snprintf(log_buffer,sizeof(log_buffer), - "setgroups failed for UID = %lu, error: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); - - log_err(errno, __func__, log_buffer); - } - - if (setegid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != PBSE_NONE) - { - snprintf(log_buffer, sizeof(log_buffer), - "setegid(%lu) failed for UID = %lu, error: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid, - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); - - log_err(errno, __func__, log_buffer); - - return(-1); - } - - if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, TRUE) != PBSE_NONE) + if (become_the_user(pjob, true) != PBSE_NONE) { - snprintf(log_buffer, sizeof(log_buffer), - "seteuid(%lu) failed, error: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); - log_err(errno, __func__, log_buffer); setegid(pbsgroup); diff --git a/src/test/prolog/scaffolding.c b/src/test/prolog/scaffolding.c index 1bc810c9fe..3675af9c67 100644 --- a/src/test/prolog/scaffolding.c +++ b/src/test/prolog/scaffolding.c @@ -67,7 +67,8 @@ void encode_used(job *pjob, int perm, Json::Value *list, tlist_head *phead) int become_the_user( - job *pjob) + job *pjob, + bool want_effective) { return(0); diff --git a/src/test/requests/scaffolding.c b/src/test/requests/scaffolding.c index 6b057706f3..ef277f485d 100644 --- a/src/test/requests/scaffolding.c +++ b/src/test/requests/scaffolding.c @@ -60,7 +60,7 @@ const char *wdir_ret; void log_event(int event, int event_class, char *func_name, char *buf) {} -int become_the_user(job *pjob) +int become_the_user(job *pjob, bool want_effective) { return(0); } diff --git a/src/test/start_exec/scaffolding.c b/src/test/start_exec/scaffolding.c index a9ad650668..982ac0c5e2 100644 --- a/src/test/start_exec/scaffolding.c +++ b/src/test/start_exec/scaffolding.c @@ -695,7 +695,10 @@ proc_stat_t *get_proc_stat(int pid) int setuid_ext(uid_t uid, int set_euid) { - return(0); + if (set_euid == TRUE) + return(seteuid(uid)); + + return(setuid(uid)); } int destroy_alps_reservation(char *reservation_id, char *apbasil_path, char *apbasil_protocol, int retries) diff --git a/src/test/start_exec/test_uut.c b/src/test/start_exec/test_uut.c index fdf57b802d..6ac14e9bee 100644 --- a/src/test/start_exec/test_uut.c +++ b/src/test/start_exec/test_uut.c @@ -7,8 +7,12 @@ #include #include #include +#include #include #include +#include +#include +#include #include "pbs_error.h" #include "pbs_nodes.h" @@ -48,6 +52,12 @@ int read_launcher_child_status(struct startjob_rtn *sjr, const char *job_id, int int process_launcher_child_status(struct startjob_rtn *sjr, const char *job_id, const char *application_name); void update_task_and_job_states_after_launch(task *ptask, job *pjob, const char *application_name); void escape_spaces(const char *str, std::string &escaped); +int become_the_user(job*, bool); + +#if NO_SPOOL_OUTPUT == 1 +int save_supplementary_group_list(int*, gid_t**); +int restore_supplementary_group_list(int, gid_t*); +#endif #ifdef PENABLE_LINUX_CGROUPS unsigned long long get_memory_limit_from_resource_list(job *pjob); @@ -645,6 +655,135 @@ START_TEST(test_get_num_nodes_ppn) } END_TEST + +#if NO_SPOOL_OUTPUT == 1 +START_TEST(test_save_supplementary_group_list) + { + int ngroups; + gid_t *group_list; + int rc; + + rc = save_supplementary_group_list(&ngroups, &group_list); + fail_unless(rc == 0); + fail_unless(ngroups > 0); + fail_unless((rc == 0) && (group_list != NULL)); + + if (rc == 0) + free(group_list); + } +END_TEST + +START_TEST(restore_supplementary_group_list_test) + { + int ngroups; + gid_t *group_list; + int rc; + + rc = restore_supplementary_group_list(0, NULL); + fail_unless(rc < 0); + + + // must be root to run the following since they call setgroups() + if (getuid() == 0) + { + int i; + int ngroups_alt; + gid_t *group_list_alt; + + // save the supplementary groups + rc = save_supplementary_group_list(&ngroups, &group_list); + fail_unless(rc == 0); + + // make a back up copy of the array + group_list_alt = (gid_t *)malloc(ngroups * sizeof(gid_t)); + fail_unless(group_list_alt != NULL); + memcpy(group_list_alt, group_list, ngroups * sizeof(gid_t)); + + // set to a smaller set + rc = setgroups(1, group_list); + fail_unless(rc == 0); + + // restore the full set + rc = restore_supplementary_group_list(ngroups, group_list); + fail_unless(rc == 0); + + // check to make sure full set restored + ngroups = getgroups(0, NULL); + group_list = (gid_t *)malloc(ngroups * sizeof(gid_t)); + fail_unless(group_list != NULL); + rc = getgroups(ngroups, group_list); + + for (i = 0; i < ngroups; i++) + fail_unless(group_list[i] == group_list_alt[i]); + + free(group_list_alt); + } + } +END_TEST +#endif + +START_TEST(test_become_the_user) + { + int rc; + job *pjob; + int uid; + int gid; + + // must be root to run this test + if (getuid() != 0) + return; + + pjob = (job *)calloc(1, sizeof(job)); + fail_unless(pjob != NULL); + + pjob->ji_grpcache = (struct grpcache *)calloc(1, sizeof(struct grpcache)); + fail_unless(pjob->ji_grpcache != NULL); + + pjob->ji_qs.ji_un.ji_momt.ji_exuid = 500; + pjob->ji_qs.ji_un.ji_momt.ji_exgid = 500; + + pjob->ji_grpcache->gc_ngroup = 1; + pjob->ji_grpcache->gc_groups[0] = 500; + + // fork so we can test the setxid/setexid calls in the child + rc = fork(); + fail_unless(rc != -1); + + if (rc > 0) + { + int status; + + // parent + wait(&status); + return; + } + + // child + + rc = become_the_user(pjob, true); + fail_unless(rc == PBSE_NONE); + + // check the group list, uid, gid + uid = geteuid(); + gid = getegid(); + fail_unless(uid != 500); + fail_unless(gid != 500); + + // put things back in place + fail_unless(seteuid(0) == 0); + fail_unless(setegid(0) == 0); + + rc = become_the_user(pjob, false); + fail_unless(rc == PBSE_NONE); + + // check the uid, gid + uid = getuid(); + gid = getgid(); + fail_unless(uid != 500); + fail_unless(gid != 500); + } +END_TEST + Suite *start_exec_suite(void) { Suite *s = suite_create("start_exec_suite methods"); @@ -699,6 +838,17 @@ Suite *start_exec_suite(void) #endif suite_add_tcase(s, tc_core); +#if NO_SPOOL_OUTPUT == 1 + tc_core = tcase_create("test_save_supplementary_group_list"); + tcase_add_test(tc_core, test_save_supplementary_group_list); + tcase_add_test(tc_core, restore_supplementary_group_list_test); + suite_add_tcase(s, tc_core); +#endif + + tc_core = tcase_create("test_become_the_user"); + tcase_add_test(tc_core, test_become_the_user); + suite_add_tcase(s, tc_core); + return s; } From 5e80237d6efa6ef545aca540ef615e7a79045c36 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 1 Dec 2016 13:43:01 -0700 Subject: [PATCH 076/312] Fix unit tests --- src/test/mail_throttler/scaffolding.c | 2 ++ src/test/requests/test_uut.c | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/test/mail_throttler/scaffolding.c b/src/test/mail_throttler/scaffolding.c index 8b13789179..c10bce7cee 100644 --- a/src/test/mail_throttler/scaffolding.c +++ b/src/test/mail_throttler/scaffolding.c @@ -1 +1,3 @@ +#include +int pthread_mutex_lock(pthread_mutex_t *mutex) { return(0); } diff --git a/src/test/requests/test_uut.c b/src/test/requests/test_uut.c index b30211fd6f..7ac0eb8767 100644 --- a/src/test/requests/test_uut.c +++ b/src/test/requests/test_uut.c @@ -21,6 +21,9 @@ START_TEST(test_determine_spooldir) std::string spooldir; const char *val_ptr; + memset(&pjob, 0, sizeof(job)); + pjob.ji_grpcache = (struct grpcache *)calloc(1, sizeof(struct grpcache)); + // If the spool directories don't match, we should get the default TNoSpoolDirList[0] = strdup("/home/bob/jobs/"); wdir_ret = "PBS_O_WORKDIR=/home/dbeer/jobs/"; From 0860040376d7ee69757368e50f3860759369c0b4 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 1 Dec 2016 15:32:47 -0700 Subject: [PATCH 077/312] TRQ-3841. Add a unit test to verify this change. --- src/test/pbsd_init/scaffolding.c | 4 ++++ src/test/pbsd_init/test_uut.c | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/test/pbsd_init/scaffolding.c b/src/test/pbsd_init/scaffolding.c index ddfc79affa..096c70617a 100644 --- a/src/test/pbsd_init/scaffolding.c +++ b/src/test/pbsd_init/scaffolding.c @@ -117,6 +117,8 @@ int evaluated; int aborted; int freed_job_allocation; int job_state; +int locked_job = 0; +int unlocked_job = 0; bool dont_find_job; bool dont_find_node; @@ -577,11 +579,13 @@ int insert_addr_name_info( int unlock_ji_mutex(job *pjob, const char *id, const char *msg, int logging) { + unlocked_job++; return(0); } int lock_ji_mutex(job *pjob, const char *id, const char *msg, int logging) { + locked_job++; return(0); } diff --git a/src/test/pbsd_init/test_uut.c b/src/test/pbsd_init/test_uut.c index 772cb59e2a..f2335a4e69 100644 --- a/src/test/pbsd_init/test_uut.c +++ b/src/test/pbsd_init/test_uut.c @@ -21,6 +21,8 @@ extern int evaluated; extern int aborted; extern int freed_job_allocation; extern int job_state; +extern int unlocked_job; +extern int locked_job; extern bool dont_find_job; extern bool dont_find_node; extern pbs_queue *allocd_queue; @@ -62,12 +64,19 @@ START_TEST(test_check_jobs_queue) sprintf(pjob.ji_qs.ji_jobid, "1.napali"); sprintf(pjob.ji_qs.ji_queue, "lost"); + unlocked_job = 0; + locked_job = 0; + check_jobs_queue(&pjob); fail_unless(allocd_queue != NULL); fail_unless(!strcmp(allocd_queue->qu_qs.qu_name, pjob.ji_qs.ji_queue)); fail_unless(allocd_queue->qu_qs.qu_type == QTYPE_Execution); fail_unless(allocd_queue->qu_attr[QA_ATR_GhostQueue].at_val.at_long == 1); fail_unless(!strcmp(allocd_queue->qu_attr[QA_ATR_QType].at_val.at_str, "Execution")); + + // check_jobs_queue should unlock and re-lock the job once each + fail_unless(unlocked_job == 1); + fail_unless(locked_job == 1); } END_TEST From 7696d2c7b55a13af7c5dbf61166e7d8e96fc062d Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 1 Dec 2016 17:00:55 -0700 Subject: [PATCH 078/312] TRQ-3879. Add a null check. --- src/resmom/mom_comm.c | 4 ++++ src/test/mom_comm/test_uut.c | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index dc4ab79b6e..055a5a83c9 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -3810,6 +3810,10 @@ int get_reply_stream( job *pjob) /* I */ { + if ((pjob == NULL) || + (pjob->ji_hosts == NULL)) + return(-1); + hnodent *np = pjob->ji_hosts; return (tcp_connect_sockaddr((struct sockaddr *)&np->sock_addr,sizeof(np->sock_addr), true)); diff --git a/src/test/mom_comm/test_uut.c b/src/test/mom_comm/test_uut.c index 02013b7df3..ca5b95c4cb 100644 --- a/src/test/mom_comm/test_uut.c +++ b/src/test/mom_comm/test_uut.c @@ -22,6 +22,7 @@ extern int log_event_counter; extern bool ms_val; extern mom_server mom_servers[PBS_MAXSERVER]; extern int ServerStatUpdateInterval; +int get_reply_stream(job *pjob); extern time_t LastServerUpdateTime; extern time_t time_now; extern bool ForceServerUpdate; @@ -46,6 +47,19 @@ int get_req_and_task_index_from_local_rank(job *pjob, int local_rank, unsigned i extern bool per_task; + +START_TEST(test_get_reply_stream) + { + job pjob; + pjob.ji_hosts = NULL; + + // Make sure we don't segfault + fail_unless(get_reply_stream(NULL) == -1); + fail_unless(get_reply_stream(&pjob) == -1); + } +END_TEST + + START_TEST(test_get_req_and_task_index_from_local_rank) { job *pjob = (job *)calloc(1, sizeof(job)); @@ -612,6 +626,7 @@ Suite *mom_comm_suite(void) tc_core = tcase_create("send_update_soon_test"); tcase_add_test(tc_core, send_update_soon_test); + tcase_add_test(tc_core, test_get_reply_stream); suite_add_tcase(s, tc_core); tc_core = tcase_create("get_stat_update_interval_test"); From dcbea49d582a170cab3758960c40aa05688ef9ff Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 2 Dec 2016 09:25:57 -0700 Subject: [PATCH 079/312] Fix unit test. --- src/test/mom_comm/test_uut.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/mom_comm/test_uut.c b/src/test/mom_comm/test_uut.c index ca5b95c4cb..d5516c62fc 100644 --- a/src/test/mom_comm/test_uut.c +++ b/src/test/mom_comm/test_uut.c @@ -626,7 +626,9 @@ Suite *mom_comm_suite(void) tc_core = tcase_create("send_update_soon_test"); tcase_add_test(tc_core, send_update_soon_test); +#ifdef PENABLE_LINUX_CGROUPS tcase_add_test(tc_core, test_get_reply_stream); +#endif suite_add_tcase(s, tc_core); tc_core = tcase_create("get_stat_update_interval_test"); From 1fe047c9fe0c75eb6447515499fa75317e6d92ef Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 2 Dec 2016 09:35:11 -0700 Subject: [PATCH 080/312] Move test_get_reply_stream out of the CGROUPS ifdef. --- src/test/mom_comm/test_uut.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/test/mom_comm/test_uut.c b/src/test/mom_comm/test_uut.c index d5516c62fc..24322773b1 100644 --- a/src/test/mom_comm/test_uut.c +++ b/src/test/mom_comm/test_uut.c @@ -48,18 +48,6 @@ int get_req_and_task_index_from_local_rank(job *pjob, int local_rank, unsigned i extern bool per_task; -START_TEST(test_get_reply_stream) - { - job pjob; - pjob.ji_hosts = NULL; - - // Make sure we don't segfault - fail_unless(get_reply_stream(NULL) == -1); - fail_unless(get_reply_stream(&pjob) == -1); - } -END_TEST - - START_TEST(test_get_req_and_task_index_from_local_rank) { job *pjob = (job *)calloc(1, sizeof(job)); @@ -85,6 +73,18 @@ END_TEST #endif +START_TEST(test_get_reply_stream) + { + job pjob; + pjob.ji_hosts = NULL; + + // Make sure we don't segfault + fail_unless(get_reply_stream(NULL) == -1); + fail_unless(get_reply_stream(&pjob) == -1); + } +END_TEST + + START_TEST(test_find_task_by_pid) { job pjob; @@ -626,9 +626,7 @@ Suite *mom_comm_suite(void) tc_core = tcase_create("send_update_soon_test"); tcase_add_test(tc_core, send_update_soon_test); -#ifdef PENABLE_LINUX_CGROUPS tcase_add_test(tc_core, test_get_reply_stream); -#endif suite_add_tcase(s, tc_core); tc_core = tcase_create("get_stat_update_interval_test"); From 60ddb3a3e53a9e5fd72eaaab8f94e84f4363feb8 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 2 Dec 2016 15:52:29 -0700 Subject: [PATCH 081/312] TRQ-3861. Add a mechanism for forcing the server to accept the new layout information from the mom. The mechanism is add -f when starting pbs_mom. Conflicts: src/lib/Libutils/machine.cpp src/server/process_mom_update.c src/test/process_mom_update/test_uut.c --- src/include/machine.hpp | 3 +++ src/lib/Libutils/machine.cpp | 15 +++++++++++++++ src/lib/Libutils/numa_chip.cpp | 17 +++++++++++++++++ src/lib/Libutils/numa_socket.cpp | 11 +++++++++++ src/resmom/mom_main.c | 9 ++++++++- src/resmom/mom_server.c | 8 ++++++++ src/server/process_mom_update.c | 23 +++++++++++++++++++---- src/test/machine/scaffolding.c | 7 +++++++ src/test/mom_server/scaffolding.c | 1 + src/test/numa_socket/scaffolding.c | 7 +++++++ src/test/process_mom_update/test_uut.c | 22 +++++++++++----------- 11 files changed, 107 insertions(+), 16 deletions(-) diff --git a/src/include/machine.hpp b/src/include/machine.hpp index 14e386e880..34cccc0f82 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -224,6 +224,7 @@ class Chip bool reserve_chip_thread(int core_index, allocation &a); bool reserve_place_thread(int core_index, allocation &a); bool reserve_chip_place_thread(int core_index, allocation &a); + void save_allocations(const Chip &other); }; @@ -287,6 +288,7 @@ class Socket void update_internal_counts(vector &allocs); int get_gpus_remaining(); int get_mics_remaining(); + void save_allocations(const Socket &other); }; @@ -355,6 +357,7 @@ class Machine void setMemory(long long mem); void addSocket(int count); // used for unit tests void setIsNuma(bool is_numa); // used for unit tests + void save_allocations(const Machine &other); void free_job_allocation(const char *jobid); int get_jobs_cpusets(const char *jobid, string &cpus, string &mems); int fit_tasks_within_sockets(req &r, allocation &job_alloc, const char *hostname, int &remaining_tasks); diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index 05985d5d87..9be3c29e00 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -1269,4 +1269,19 @@ bool Machine::is_initialized() const +void Machine::save_allocations( + + const Machine &other) + + { + this->allocations = other.allocations; + + for (size_t s = 0; s < other.sockets.size() && s < this->sockets.size(); s++) + { + this->sockets[s].save_allocations(other.sockets[s]); + } + } // END save_allocations() + + + #endif /* PENABLE_LINUX_CGROUPS */ diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index 41d21c342c..c010ca9384 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -2592,4 +2592,21 @@ bool Chip::has_socket_exclusive_allocation() const } // END has_socket_exclusive_allocation() + +/* + * Preserves all relevant information from the allocations for Chip other on this Chip + * This is called when replacing one Machine object with another, but we are trying to + * keep the information about running jobs. + */ + +void Chip::save_allocations( + + const Chip &other) + + { + this->allocations = other.allocations; + this->adjust_open_resources(); + } // END save_allocations() + + #endif /* PENABLE_LINUX_CGROUPS */ diff --git a/src/lib/Libutils/numa_socket.cpp b/src/lib/Libutils/numa_socket.cpp index 492d56213b..100ad177a3 100644 --- a/src/lib/Libutils/numa_socket.cpp +++ b/src/lib/Libutils/numa_socket.cpp @@ -951,4 +951,15 @@ int Socket::get_mics_remaining() } + +void Socket::save_allocations( + + const Socket &other) + + { + for (size_t c = 0; c < this->chips.size() && c < other.chips.size(); c++) + this->chips[c].save_allocations(other.chips[c]); + } + + #endif /* PENABLE_LINUX_CGROUPS */ diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index b52e453a3e..9f7648d5b8 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -344,6 +344,7 @@ char *path_log; int LOGLEVEL = 0; /* valid values (0 - 10) */ int DEBUGMODE = 0; bool daemonize_mom = true; +bool force_layout_update = false; long TJobStartTimeout = PBS_PROLOG_TIME; /* seconds to wait for job to launch before purging */ @@ -4226,7 +4227,7 @@ void parse_command_line( errflg = 0; - while ((c = getopt(argc, argv, "a:A:c:C:d:DFhH:l:L:mM:pPqrR:s:S:vwx-:")) != -1) + while ((c = getopt(argc, argv, "a:A:c:C:d:DfFhH:l:L:mM:pPqrR:s:S:vwx-:")) != -1) { switch (c) { @@ -4334,6 +4335,12 @@ void parse_command_line( break; + case 'f': // force layout update + + force_layout_update = true; + + break; + case 'F': /* do not fork */ // use when running under systemd diff --git a/src/resmom/mom_server.c b/src/resmom/mom_server.c index 88d7542517..f4abc5507b 100644 --- a/src/resmom/mom_server.c +++ b/src/resmom/mom_server.c @@ -309,6 +309,7 @@ extern mom_hierarchy_t *mh; extern char *stat_string_aggregate; extern unsigned int ssa_index; extern u_long localaddr; +extern bool force_layout_update; extern container::item_container received_statuses; std::vector global_gpu_status; std::vector mom_status; @@ -700,6 +701,11 @@ void gen_layout( std::vector &status) { + if (force_layout_update == true) + { + status.push_back("force_layout_update"); + } + std::stringstream layout; layout << name << "="; this_node.displayAsJson(layout, false); @@ -1868,6 +1874,8 @@ void mom_server_all_update_stat(void) num_stat_update_failures++; else { + force_layout_update = false; + num_stat_update_failures = 0; for (int sindex = 0; sindex < PBS_MAXSERVER; sindex++) { diff --git a/src/server/process_mom_update.c b/src/server/process_mom_update.c index fb0d59dee3..8798fa5a3f 100644 --- a/src/server/process_mom_update.c +++ b/src/server/process_mom_update.c @@ -749,13 +749,15 @@ int save_node_status( void update_layout_if_needed( pbsnode *pnode, - const std::string &layout) + const std::string &layout, + bool force) { char log_buf[LOCAL_LOG_BUF_SIZE]; std::vector valid_ids; - if (pnode->nd_layout.is_initialized() == false) + if ((pnode->nd_layout.is_initialized() == false) + (force == true)) { for (size_t i = 0; i < pnode->nd_job_usages.size(); i++) { @@ -765,7 +767,12 @@ void update_layout_if_needed( valid_ids.push_back(id); } - pnode->nd_layout.reinitialize_from_json(layout, valid_ids); + Machine m(layout, valid_ids); + + if (pnode->nd_layout.is_initialized() == true) + m.save_allocations(pnode->nd_layout); + + pnode->nd_layout = m; } else if ((pnode->nd_layout.getTotalThreads() != pnode->nd_slots.get_total_execution_slots()) && (pnode->nd_job_usages.size() == 0)) @@ -824,6 +831,9 @@ int process_status_info( int rc = PBSE_NONE; bool send_hello = false; std::string temp; +#ifdef PENABLE_LINUX_CGROUPS + bool force_layout_update = false; +#endif get_svr_attr_b(SRV_ATR_MomJobSync, &mom_job_sync); get_svr_attr_b(SRV_ATR_AutoNodeNP, &auto_np); @@ -913,10 +923,15 @@ int process_status_info( str = status_info[i].c_str(); } #ifdef PENABLE_LINUX_CGROUPS + else if (!strcmp(str, "force_layout_update")) + force_layout_update = true; else if (!strncmp(str, "layout", 6)) { // Add 7 to skip "layout=" - update_layout_if_needed(current, str + 7); + update_layout_if_needed(current, str + 7, force_layout_update); + + // reset this to false in case we have a mom hierarchy in place + force_layout_update = false; continue; } diff --git a/src/test/machine/scaffolding.c b/src/test/machine/scaffolding.c index 4c41501ddb..333f52a37e 100644 --- a/src/test/machine/scaffolding.c +++ b/src/test/machine/scaffolding.c @@ -242,6 +242,13 @@ bool Socket::is_available() const void Socket::displayAsJson(Json::Value &out, bool jobs) const {} +void Socket::save_allocations(const Socket &other) {} + +int Socket::get_total_gpus() const + { + return(0); + } + unsigned long req::getMemory() const { return(req_mem); diff --git a/src/test/mom_server/scaffolding.c b/src/test/mom_server/scaffolding.c index 17a306cf64..5892d42262 100644 --- a/src/test/mom_server/scaffolding.c +++ b/src/test/mom_server/scaffolding.c @@ -26,6 +26,7 @@ #define MAXLINE 1024 +bool force_layout_update = false; std::list alljobs_list; char log_buffer[LOG_BUF_SIZE]; char *apbasil_protocol = NULL; diff --git a/src/test/numa_socket/scaffolding.c b/src/test/numa_socket/scaffolding.c index 1a06b9532e..e5219ef6e3 100644 --- a/src/test/numa_socket/scaffolding.c +++ b/src/test/numa_socket/scaffolding.c @@ -128,6 +128,8 @@ Chip::~Chip() void Chip::aggregate_allocations(std::vector &master_list) {} +void Chip::save_allocations(const Chip &other) {} + void Chip::displayAsJson( Json::Value &out, @@ -136,6 +138,11 @@ void Chip::displayAsJson( { } +int Chip::get_total_gpus() const + { + return(0); + } + bool Chip::spread_place( diff --git a/src/test/process_mom_update/test_uut.c b/src/test/process_mom_update/test_uut.c index aef55b28d5..8eed741992 100644 --- a/src/test/process_mom_update/test_uut.c +++ b/src/test/process_mom_update/test_uut.c @@ -12,7 +12,7 @@ int set_note_error(struct pbsnode *np, const char *str); int restore_note(struct pbsnode *np); #ifdef PENABLE_LINUX_CGROUPS -void update_layout_if_needed(pbsnode *pnode, const std::string &layout); +void update_layout_if_needed(pbsnode *pnode, const std::string &layout, bool force); extern int event_logged; @@ -26,7 +26,7 @@ START_TEST(test_update_layout_if_needed) fail_unless(pnode.nd_layout.is_initialized() == false); event_logged = 0; - update_layout_if_needed(&pnode, sans_threads); + update_layout_if_needed(&pnode, sans_threads, false); fail_unless(event_logged == 0); fail_unless(pnode.nd_layout.is_initialized() != false); @@ -35,10 +35,10 @@ START_TEST(test_update_layout_if_needed) pnode.nd_slots.add_execution_slot(); // Calling again without changing the number of slots should do nothing - update_layout_if_needed(&pnode, sans_threads); - update_layout_if_needed(&pnode, sans_threads); - update_layout_if_needed(&pnode, sans_threads); - update_layout_if_needed(&pnode, sans_threads); + update_layout_if_needed(&pnode, sans_threads, false); + update_layout_if_needed(&pnode, sans_threads, false); + update_layout_if_needed(&pnode, sans_threads, false); + update_layout_if_needed(&pnode, sans_threads, false); fail_unless(event_logged == 0); // Simulate that threads were turned on in this machine, therefore increasing the number of @@ -46,15 +46,15 @@ START_TEST(test_update_layout_if_needed) for (int i = 0; i < 16; i++) pnode.nd_slots.add_execution_slot(); - update_layout_if_needed(&pnode, with_threads); + update_layout_if_needed(&pnode, with_threads, false); fail_unless(event_logged == 1); // Calling again without changing the number of slots should do nothing - update_layout_if_needed(&pnode, with_threads); - update_layout_if_needed(&pnode, with_threads); - update_layout_if_needed(&pnode, with_threads); - update_layout_if_needed(&pnode, with_threads); + update_layout_if_needed(&pnode, with_threads, false); + update_layout_if_needed(&pnode, with_threads, false); + update_layout_if_needed(&pnode, with_threads, false); + update_layout_if_needed(&pnode, with_threads, false); fail_unless(event_logged == 1, "event logged: %d", event_logged); } END_TEST From 001b2e659710fb1b5c7195c9e1c345cbde772f24 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 2 Dec 2016 16:03:35 -0700 Subject: [PATCH 082/312] TRQ-3861. Fix an error in the cherry-pick. --- src/server/process_mom_update.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/process_mom_update.c b/src/server/process_mom_update.c index 8798fa5a3f..15c19a7924 100644 --- a/src/server/process_mom_update.c +++ b/src/server/process_mom_update.c @@ -756,7 +756,7 @@ void update_layout_if_needed( char log_buf[LOCAL_LOG_BUF_SIZE]; std::vector valid_ids; - if ((pnode->nd_layout.is_initialized() == false) + if ((pnode->nd_layout.is_initialized() == false) || (force == true)) { for (size_t i = 0; i < pnode->nd_job_usages.size(); i++) From 0575370134223666285da39617d313e8a85673c3 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 2 Dec 2016 16:16:01 -0700 Subject: [PATCH 083/312] Fix another scaffolding error. --- src/test/process_mom_update/scaffolding.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/process_mom_update/scaffolding.c b/src/test/process_mom_update/scaffolding.c index 4569395967..ca7d614503 100644 --- a/src/test/process_mom_update/scaffolding.c +++ b/src/test/process_mom_update/scaffolding.c @@ -650,6 +650,8 @@ Chip::~Chip() {} Core::Core() {} Core::~Core() {} +void Machine::save_allocations(const Machine &other) {} + bool Machine::is_initialized() const { return(this->initialized); From c42e91d4a531dd5a709d454558ad2cadbd86014a Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 2 Dec 2016 16:29:04 -0700 Subject: [PATCH 084/312] Revert "TRQ-3682. Fix gpu indexing for multi-gpu nodes." This reverts commit 06664a7c9964fba1b3665e44e64ae9b4a05243d6. Conflicts: src/resmom/nvidia.c src/resmom/start_exec.c --- src/resmom/nvidia.c | 113 --------------------------------- src/resmom/parse_config.c | 8 --- src/resmom/start_exec.c | 30 ++------- src/test/start_exec/test_uut.c | 31 ++------- 4 files changed, 12 insertions(+), 170 deletions(-) diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index 8e420dde96..b0eb4d4779 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -151,14 +151,6 @@ extern nodeboard node_boards[]; #endif -// Default to version 7.0 -extern int cuda_version; -std::string nvcc_version_cmd("/usr/local/cuda/bin/nvcc --version"); - -// Forward declaration -void set_cuda_version(); - - /* * Function to log nvml errors */ @@ -493,9 +485,6 @@ int check_nvidia_setup() nvidia_setup_is_ok = TRUE; #endif /* NVML_API */ } - - set_cuda_version(); - return (nvidia_setup_is_ok); } @@ -2215,105 +2204,3 @@ int add_gpu_status( } /* END add_gpu_status() */ - -/* - * matches_version_string() - * - * Sets the global variable cuda_version from versiong_string, if possible - * - * @param version_string - a string that should match the format number.number.number - * @return true if the string matched the appropriate format and the version was set - */ - -bool matches_version_string( - - char *version_string) - - { - char *ptr = version_string; - char *end = ptr; - int tmp_version; - bool match = false; - - tmp_version = strtol(ptr, &end, 10) * 10; - - if (*end == '.') - { - ptr = end + 1; - - tmp_version += strtol(ptr, &end, 10); - - if (*end == '.') - { - // valid format - match = true; - cuda_version = tmp_version; - } - } - - return(match); - } // END matches_version_string() - - - -/* - * set_cuda_version() - * - * Determines what version of CUDA is installed on this machine - */ - -void set_cuda_version() - { - char buf[MAXLINE * 4]; - FILE *nvcc_pipe; - char *ptr = buf; - int bytes_read = 0; - bool version_set = false; - - if ((nvcc_pipe = popen(nvcc_version_cmd.c_str(), "r")) == NULL) - { - snprintf(log_buffer, sizeof(log_buffer), - "Cannot open pipe to get CUDA version with command '%s' - assuming version %d", - nvcc_version_cmd.c_str(), cuda_version); - log_err(errno, __func__, log_buffer); - } - - memset(buf, 0, sizeof(buf)); - int fd = fileno(nvcc_pipe); - - while ((version_set == false) && - ((bytes_read = read(fd, ptr, sizeof(buf) - 1)) > 0)) - { - char *tmpptr = ptr; - - // Version string appears in the format V.., so find the V - // and check if the rest matches - while ((tmpptr = strrchr(tmpptr, 'V')) != NULL) - { - if (matches_version_string(tmpptr + 1) == true) - { - version_set = true; - break; - } - - // At the beginning - if (tmpptr == ptr) - break; - else - tmpptr--; - } - } - - pclose(nvcc_pipe); - - if (version_set == false) - { - snprintf(log_buffer, sizeof(log_buffer), - "Couldn't find a CUDA version string with command '%s', defaulting to version %d", - nvcc_version_cmd.c_str(), cuda_version); - log_err(-1, __func__, log_buffer); - } - - } // END set_cuda_version() - - diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index 1a05e3df3c..2a315cbd71 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -165,13 +165,7 @@ char **maskclient = NULL; /* wildcard connections */ char MOMConfigVersion[64]; int MOMConfigDownOnError = 0; int MOMConfigRestart = 0; -// Setting this variable is redundant for cgroups, so we'll default to false -// It can only cause us pain. -#ifdef PENABLE_LINUX_CGROUPS -int MOMCudaVisibleDevices = 0; -#else int MOMCudaVisibleDevices = 1; -#endif double wallfactor = 1.00; std::vector pcphosts; long pe_alarm_time = PBS_PROLOG_TIME; @@ -3314,8 +3308,6 @@ const char *validuser( return("no"); } /* END validuser() */ - - u_long setcudavisibledevices( const char *value) /* I */ diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index c7723771a2..d3daab21cd 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -252,9 +252,6 @@ extern int numa_index; extern int linux_time; -// Default to version 7.0 -int cuda_version = 70; - /* Local Variables */ static int script_in; /* script file, will be stdin */ @@ -1357,6 +1354,7 @@ int TMakeTmpDir( + /* * get_indices_from_exec_str * @@ -1373,17 +1371,13 @@ int TMakeTmpDir( * @param exec_str - the string we're parsing * @param buf - where the list of absolute mic indices goes * @param buf_size - maximum size that can be written into buf - * @param param_cuda_version - -1 if we're not doing gpus, or the version of cuda otherwise - * If we are doing cuda, and we're above version 7.0, we ignore the index - * in the string and count from 0. */ int get_indices_from_exec_str( - const char *exec_str, - char *buf, - int buf_size, - int param_cuda_version) + const char *exec_str, /* I */ + char *buf, /* O */ + int buf_size) /* I */ { char *work_str; @@ -1396,7 +1390,6 @@ int get_indices_from_exec_str( #endif int numa_offset = 0; int index; - int relative_count = 0; if ((buf == NULL) || (exec_str == NULL)) @@ -1438,19 +1431,12 @@ int get_indices_from_exec_str( if ((slash = strchr(tok, '/')) != NULL) { - // CUDA versions 7.0 and greater always index from 0 on up for the program, so if you - // have gpus 2 and 3, we should pass 0 and 1 - if (param_cuda_version >= 70) - index = relative_count; - else - index = strtol(slash+1, NULL, 10) + numa_offset; + index = strtol(slash+1, NULL, 10) + numa_offset; if (buf[0] != '\0') snprintf(buf + strlen(buf), buf_size - strlen(buf), ",%d", index); else snprintf(buf, buf_size, "%d", index); - - relative_count++; } } @@ -1464,7 +1450,6 @@ int get_indices_from_exec_str( } /* END get_indices_from_exec_str() */ - /* * get_num_nodes_ppn * @@ -1783,7 +1768,7 @@ int InitUserEnv( if (pjob->ji_wattr[JOB_ATR_exec_mics].at_val.at_str != NULL) { - get_indices_from_exec_str(pjob->ji_wattr[JOB_ATR_exec_mics].at_val.at_str, buf, sizeof(buf), -1); + get_indices_from_exec_str(pjob->ji_wattr[JOB_ATR_exec_mics].at_val.at_str, buf, sizeof(buf)); bld_env_variables(&vtable, variables_else[tveOffloadDevices], buf); } @@ -1791,8 +1776,7 @@ int InitUserEnv( { if (MOMCudaVisibleDevices) { - get_indices_from_exec_str(pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str, buf, sizeof(buf), - cuda_version); + get_indices_from_exec_str(pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str, buf, sizeof(buf)); bld_env_variables(&vtable, variables_else[tveCudaVisibleDevices], buf); } } diff --git a/src/test/start_exec/test_uut.c b/src/test/start_exec/test_uut.c index 6ac14e9bee..56409588ee 100644 --- a/src/test/start_exec/test_uut.c +++ b/src/test/start_exec/test_uut.c @@ -19,7 +19,7 @@ #include "test_uut.h" int job_nodes(job &pjob); -int get_indices_from_exec_str(const char *exec_str, char *buf, int buf_size, int cuda_version); +int get_indices_from_exec_str(const char *exec_str, char *buf, int buf_size); int remove_leading_hostname(char **jobpath); int get_num_nodes_ppn(const char*, int*, int*); int setup_process_launch_pipes(int &kid_read, int &kid_write, int &parent_read, int &parent_write); @@ -505,40 +505,19 @@ START_TEST(test_get_indices_from_exec_str) { char buf[1024]; - fail_unless(get_indices_from_exec_str(NULL, NULL, 0, -1) != PBSE_NONE); + fail_unless(get_indices_from_exec_str(NULL, NULL, 0) != PBSE_NONE); strcpy(mom_alias, "slesmic"); - fail_unless(get_indices_from_exec_str("slesmic-0-mic/1+slesmic-0-mic/0", - buf, - sizeof(buf), - -1) == PBSE_NONE); + fail_unless(get_indices_from_exec_str("slesmic-0-mic/1+slesmic-0-mic/0", buf, sizeof(buf)) == PBSE_NONE); fail_unless(!strcmp(buf, "1,0")); strcpy(mom_alias, "napali"); - fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3", - buf, - sizeof(buf), - -1) == PBSE_NONE); + fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3", buf, sizeof(buf)) == PBSE_NONE); fail_unless(!strcmp(buf, "1,2,3"), buf); - - fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3", - buf, - sizeof(buf), - 70) == PBSE_NONE); - fail_unless(!strcmp(buf, "0,1,2"), buf); - - fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3", - buf, - sizeof(buf), - 80) == PBSE_NONE); - fail_unless(!strcmp(buf, "0,1,2"), buf); - fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3+waimea-gpu/0+waimea-gpu/1+waimea-gpu/2", - buf, - sizeof(buf), - -1) == PBSE_NONE); + fail_unless(get_indices_from_exec_str("napali-gpu/1+napali-gpu/2+napali-gpu/3+waimea-gpu/0+waimea-gpu/1+waimea-gpu/2", buf, sizeof(buf)) == PBSE_NONE); fail_unless(!strcmp(buf, "1,2,3"), buf); } END_TEST From 6899ae18b65b6f73f8aacefc353949e1caefdfa2 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 2 Dec 2016 16:27:07 -0700 Subject: [PATCH 085/312] TRQ-3682. Add back that we don't set CUDA_VISIBLE_DEVICES when cgroups are enabled. --- src/resmom/parse_config.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index 2a315cbd71..ad2ba61a84 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -165,7 +165,13 @@ char **maskclient = NULL; /* wildcard connections */ char MOMConfigVersion[64]; int MOMConfigDownOnError = 0; int MOMConfigRestart = 0; +// Setting this variable is redundant for cgroups, so we'll default to false +// It can only cause us pain. +#ifdef PENABLE_LINUX_CGROUPS +int MOMCudaVisibleDevices = 0; +#else int MOMCudaVisibleDevices = 1; +#endif double wallfactor = 1.00; std::vector pcphosts; long pe_alarm_time = PBS_PROLOG_TIME; From c110cd693821badfb2ba20709604fc9e3bbe6232 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 2 Dec 2016 16:53:40 -0700 Subject: [PATCH 086/312] TRQ-3798. Make our decimal division for cgroup memory more precise. Conflicts: src/test/numa_chip/scaffolding.c --- src/include/machine.hpp | 4 ++-- src/lib/Libutils/numa_chip.cpp | 18 +++++++++--------- src/lib/Libutils/numa_socket.cpp | 4 ++-- src/test/machine/scaffolding.c | 2 +- src/test/numa_chip/scaffolding.c | 2 +- src/test/numa_chip/test_uut.c | 13 +++++++++++++ src/test/numa_socket/scaffolding.c | 2 +- 7 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/include/machine.hpp b/src/include/machine.hpp index 34cccc0f82..6383bb767d 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -184,7 +184,7 @@ class Chip void setCores(int cores); // used for unit tests void setThreads(int threads); // used for unit tests void setChipAvailable(bool available); - float how_many_tasks_fit(const req &r, int place_type) const; + double how_many_tasks_fit(const req &r, int place_type) const; bool has_socket_exclusive_allocation() const; bool task_will_fit(const req &r, int place_type) const; int free_core_count() const; @@ -274,7 +274,7 @@ class Socket void displayAsJson(Json::Value &out, bool include_jobs) const; void setId(int id); void addChip(); // used for unit tests - float how_many_tasks_fit(const req &r, int place_type) const; + double how_many_tasks_fit(const req &r, int place_type) const; void place_all_execution_slots(req &r, allocation &task_alloc); bool spread_place(req &r, allocation &master, int execution_slots_per, int &remainder, bool chips); bool spread_place_pu(req &r, allocation &task_alloc, int &cores, int &lprocs, int &gpus, int &mics); diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index c010ca9384..1b8f8a96a2 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -930,16 +930,16 @@ int Chip::free_core_count() const * @return the number of tasks that fit. This can be 0 */ -float Chip::how_many_tasks_fit( +double Chip::how_many_tasks_fit( const req &r, int place_type) const { - float cpu_tasks; - float gpu_tasks; - float mic_tasks; - float mem_tasks = 0; + double cpu_tasks; + double gpu_tasks; + double mic_tasks; + double mem_tasks = 0; // Consider exclusive socket and node the same as exclusive chip for our purposes if ((place_type == exclusive_socket) || @@ -951,7 +951,7 @@ float Chip::how_many_tasks_fit( (this->chipIsAvailable()) == true)) { // Need to handle place={core|thread}[=x] - float max_cpus = r.getExecutionSlots(); + double max_cpus = r.getExecutionSlots(); if (r.getPlaceCores() > 0) max_cpus = r.getPlaceCores(); else if (r.getPlaceThreads() > 0) @@ -964,7 +964,7 @@ float Chip::how_many_tasks_fit( else cpu_tasks = this->availableThreads / max_cpus; - long long memory = r.getMemory(); + unsigned long memory = r.getMemory(); // Memory isn't required for submission if (memory != 0) @@ -978,7 +978,7 @@ float Chip::how_many_tasks_fit( else mem_tasks = cpu_tasks; - float gpus = r.getGpus(); + double gpus = r.getGpus(); if (gpus > 0) { gpu_tasks = this->available_gpus / gpus; @@ -986,7 +986,7 @@ float Chip::how_many_tasks_fit( mem_tasks = gpu_tasks; } - float mics = r.getMics(); + double mics = r.getMics(); if (mics > 0) { mic_tasks = this->available_mics / mics; diff --git a/src/lib/Libutils/numa_socket.cpp b/src/lib/Libutils/numa_socket.cpp index 100ad177a3..d5915e5986 100644 --- a/src/lib/Libutils/numa_socket.cpp +++ b/src/lib/Libutils/numa_socket.cpp @@ -487,13 +487,13 @@ void Socket::update_internal_counts( * @return - the number of tasks from r that could be placed on this socket */ -float Socket::how_many_tasks_fit( +double Socket::how_many_tasks_fit( const req &r, int place_type) const { - float num_that_fit = 0; + double num_that_fit = 0; if ((this->socket_exclusive == false) && ((place_type != exclusive_socket) || diff --git a/src/test/machine/scaffolding.c b/src/test/machine/scaffolding.c index 333f52a37e..32a792e8a5 100644 --- a/src/test/machine/scaffolding.c +++ b/src/test/machine/scaffolding.c @@ -207,7 +207,7 @@ int Socket::getAvailableChips() const return(1); } -float Socket::how_many_tasks_fit(const req &r, int place_type) const +double Socket::how_many_tasks_fit(const req &r, int place_type) const { return(num_tasks_fit); diff --git a/src/test/numa_chip/scaffolding.c b/src/test/numa_chip/scaffolding.c index 0ce3f894c0..09a71bf45d 100644 --- a/src/test/numa_chip/scaffolding.c +++ b/src/test/numa_chip/scaffolding.c @@ -175,7 +175,7 @@ int req::set_value(const char *name, const char *value, bool is_default) if (!strcmp(name, "lprocs")) this->execution_slots = atoi(value); else if (!strcmp(name, "memory")) - this->mem = atoi(value); + this->mem = strtol(value, NULL, 10); else if (!strcmp(name, GPUS)) this->gpus = atoi(value); else if (!strcmp(name, MICS)) diff --git a/src/test/numa_chip/test_uut.c b/src/test/numa_chip/test_uut.c index 98cd19229a..9ea16b491c 100644 --- a/src/test/numa_chip/test_uut.c +++ b/src/test/numa_chip/test_uut.c @@ -774,6 +774,19 @@ START_TEST(test_how_many_tasks_fit) fail_unless(c.how_many_tasks_fit(r2, 0) == 5); r2.set_value("mics", "1", false); fail_unless(c.how_many_tasks_fit(r2, 0) == 0); + + Chip large_mem; + large_mem.setThreads(8); + large_mem.setMemory(128849018880); // 256 GB + large_mem.setCores(8); + large_mem.setChipAvailable(true); + for (int i = 0; i < 8; i++) + large_mem.make_core(i); + req r3; + r3.set_value("lprocs", "8", false); + r3.set_value("memory", "128849018880kb", false); // 256 gb + float fitting_tasks = large_mem.how_many_tasks_fit(r3, exclusive_none); + fail_unless(fitting_tasks == 1.0); } END_TEST diff --git a/src/test/numa_socket/scaffolding.c b/src/test/numa_socket/scaffolding.c index e5219ef6e3..3cf030ab55 100644 --- a/src/test/numa_socket/scaffolding.c +++ b/src/test/numa_socket/scaffolding.c @@ -258,7 +258,7 @@ int get_machine_total_memory(hwloc_topology_t topology, unsigned long *memory) return(PBSE_NONE); } -float Chip::how_many_tasks_fit(req const &r, int place_type) const +double Chip::how_many_tasks_fit(req const &r, int place_type) const { return(tasks); } From 4b5a8e3cc14ff01e90ba8aad51206d62cc4b98c2 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 6 Dec 2016 11:30:05 -0700 Subject: [PATCH 087/312] TRQ-3735. Fix dependencies when using a job suffix alias. Conflicts: src/server/array_func.c src/server/req_modify.c src/test/array_func/scaffolding.c --- src/server/array_func.c | 47 ++++------- src/server/job_container.c | 124 +++++++--------------------- src/server/req_modify.c | 25 +++--- src/server/req_register.c | 18 ++-- src/test/array_func/scaffolding.c | 5 +- src/test/job_container/test_uut.c | 7 +- src/test/req_modify/scaffolding.c | 14 ++-- src/test/req_modify/test_uut.c | 6 +- src/test/req_register/scaffolding.c | 7 +- 9 files changed, 91 insertions(+), 162 deletions(-) diff --git a/src/server/array_func.c b/src/server/array_func.c index 0677864860..a1cf0d88b6 100644 --- a/src/server/array_func.c +++ b/src/server/array_func.c @@ -61,7 +61,7 @@ extern int array_upgrade(job_array *, int, int, int *); -extern char *get_correct_jobname(const char *jobid); +extern const char *get_correct_jobname(const char *jobid, std::string &correct); extern int count_user_queued_jobs(pbs_queue *,char *); /* global data items used */ @@ -95,25 +95,16 @@ int is_array( char *id) { - int rc = FALSE; + int rc = FALSE; - char *bracket_ptr; - char *end_bracket_ptr; - char *tmpjobid; - char jobid[PBS_MAXSVRJOBID]; - char temp_jobid[PBS_MAXSVRJOBID]; + char *bracket_ptr; + char *end_bracket_ptr; + char jobid[PBS_MAXSVRJOBID]; + char temp_jobid[PBS_MAXSVRJOBID]; + std::string tmpjobid; - tmpjobid = get_correct_jobname(id); - if (tmpjobid == NULL) - { - /* Maybe we should just return ENOMEM? */ - snprintf(jobid, sizeof(jobid), "%s", id); - } - else - { - snprintf(jobid, sizeof(jobid), "%s", tmpjobid); - free(tmpjobid); - } + get_correct_jobname(id, tmpjobid); + snprintf(jobid, sizeof(jobid), "%s", tmpjobid.c_str()); /* Check to see if we have an array dependency */ /* If there is an array dependency count then we will */ @@ -173,12 +164,10 @@ job_array *get_array( const char *id) { - job_array *pa; - char *tmpjobid; + job_array *pa; + std::string tmpjobid; - tmpjobid = get_correct_jobname(id); - if (tmpjobid == NULL) - return(NULL); + get_correct_jobname(id, tmpjobid); allarrays.lock(); @@ -189,8 +178,6 @@ job_array *get_array( allarrays.unlock(); - free(tmpjobid); - return(pa); } /* END get_array() */ @@ -211,12 +198,10 @@ job_array *get_and_remove_array( const char *id) { - job_array *pa = NULL; - char *tmpjobid; + job_array *pa = NULL; + std::string tmpjobid; - tmpjobid = get_correct_jobname(id); - if (tmpjobid == NULL) - return(NULL); + get_correct_jobname(id, tmpjobid); allarrays.lock(); @@ -230,8 +215,6 @@ job_array *get_and_remove_array( allarrays.unlock(); - free(tmpjobid); - return(pa); } // END get_and_remove_array() diff --git a/src/server/job_container.c b/src/server/job_container.c index bbd44e31e4..6cd690a01e 100644 --- a/src/server/job_container.c +++ b/src/server/job_container.c @@ -111,14 +111,15 @@ id_map job_mapper; * * allocs the correct job name * @param jobid (I) - the jobid as passed in (NUM.SERVER_NAME) - * @return a pointer to the correct job name (alloc'd) + * @param correct (O) - the correct job id as a string object + * @return a pointer to the correct job name */ -char *get_correct_jobname( +const char *get_correct_jobname( - const char *jobid) /* I */ + const char *jobid, + std::string &correct) { - char *correct = NULL; char *dot; char *work_jobid = strdup(jobid); /* first suffix could be the server name or the alias */ @@ -161,47 +162,31 @@ char *get_correct_jobname( { if (strcmp(second_suffix,alias) == 0) { - correct = strdup(jobid); + correct = jobid; - if (correct == NULL) - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(correct); + return(correct.c_str()); } } else if (first_suffix == NULL) { /* alloc memory and sprint, add 3 for 2 '.' and NULL terminator */ len = strlen(work_jobid) + strlen(server_name) + strlen(alias) + 3; - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s.%s", - work_jobid,server_name,alias); + correct = work_jobid; + correct += "."; + correct += server_name; + correct += "."; + correct += alias; } else { /* add 2 for null terminator and '.' */ len = strlen(alias) + 2 + strlen(work_jobid); - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s",work_jobid,alias); + correct = work_jobid; + correct += "."; + correct += alias; } } /* END if (server_suffix && alias) */ else if (server_suffix == TRUE) @@ -210,30 +195,15 @@ char *get_correct_jobname( if (first_suffix != NULL) { - correct = strdup(work_jobid); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } + correct = work_jobid; } else { len = strlen(work_jobid) + strlen(server_name) + 2; - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s", - work_jobid,server_name); + correct = work_jobid; + correct += "."; + correct += server_name; } } /* END if (just server_suffix) */ else if (alias != NULL) @@ -244,16 +214,9 @@ char *get_correct_jobname( { len = strlen(work_jobid) + strlen(alias) + 2; - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s",work_jobid,alias); + correct = work_jobid; + correct += "."; + correct += alias; } else { @@ -263,18 +226,10 @@ char *get_correct_jobname( *dot = '\0'; len += strlen(work_jobid); - correct = (char *)calloc(1, len); - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s", - work_jobid, - alias); + correct = work_jobid; + correct += "."; + correct += alias; *dot = '.'; } @@ -289,16 +244,8 @@ char *get_correct_jobname( } len = strlen(work_jobid) + 1; - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - snprintf(correct,len,"%s",work_jobid); + correct = work_jobid; if (first_suffix != NULL) *dot = '.'; @@ -306,7 +253,7 @@ char *get_correct_jobname( free(work_jobid); - return(correct); + return(correct.c_str()); } /* END get_correct_jobname() */ @@ -389,8 +336,6 @@ job *svr_find_job( { char *at; - char *comp = NULL; - int different = FALSE; char *dash = NULL; char *dot = NULL; char without_dash[PBS_MAXSVRJOBID + 1]; @@ -399,6 +344,8 @@ job *svr_find_job( job *pj = NULL; + std::string comp; + if (NULL == jobid) { log_err(-1,__func__,"jobid is null"); @@ -440,11 +387,7 @@ job *svr_find_job( if ((is_svr_attr_set(SRV_ATR_display_job_server_suffix)) || (is_svr_attr_set(SRV_ATR_job_suffix_alias))) { - comp = get_correct_jobname(jid_ptr); - different = TRUE; - - if (comp == NULL) - return(NULL); + get_correct_jobname(jid_ptr, comp); } else { @@ -456,7 +399,7 @@ job *svr_find_job( /* if we're searching for the external we want find_job_by_array to * return the parent, but if we're searching for the cray subjob then * we want find_job_by_array to return the sub job */ - pj = find_job_by_array(&alljobs, comp, (dash != NULL) ? FALSE : get_subjob, false); + pj = find_job_by_array(&alljobs, comp.c_str(), (dash != NULL) ? FALSE : get_subjob, false); } /* when remotely routing jobs, they are removed from the @@ -466,7 +409,7 @@ job *svr_find_job( if (pj == NULL) { /* see the comment on the above call to find_job_by_array() */ - pj = find_job_by_array(&array_summary, comp, (dash != NULL) ? FALSE : get_subjob, false); + pj = find_job_by_array(&array_summary, comp.c_str(), (dash != NULL) ? FALSE : get_subjob, false); } if (at) @@ -500,9 +443,6 @@ job *svr_find_job( } } - if (different) - free(comp); - return(pj); /* may be NULL */ } /* END svr_find_job() */ diff --git a/src/server/req_modify.c b/src/server/req_modify.c index 2e5337e1ad..cadf2fd3f8 100644 --- a/src/server/req_modify.c +++ b/src/server/req_modify.c @@ -143,7 +143,7 @@ extern void rel_resc(job *); extern job *chk_job_request(char *, struct batch_request *); extern struct batch_request *cpy_checkpoint(struct batch_request *, job *, enum job_atr, int); -extern char *get_correct_jobname(const char *jobid); +extern const char *get_correct_jobname(const char *jobid, std::string &correct); /* prototypes */ void post_modify_arrayreq(batch_request *preq); @@ -967,9 +967,9 @@ void *req_modifyjob( batch_request *preq) /* I */ { - job *pjob; - svrattrl *plist; - char *correct_jobname_p; + job *pjob; + svrattrl *plist; + std::string correct_jobname; pjob = chk_job_request(preq->rq_ind.rq_modify.rq_objname, preq); @@ -979,22 +979,17 @@ void *req_modifyjob( } // get the correct job name - correct_jobname_p = get_correct_jobname(preq->rq_ind.rq_modify.rq_objname); + get_correct_jobname(preq->rq_ind.rq_modify.rq_objname, correct_jobname); // if correct job name and one passed in don't match, adjust one passed in // so that mom will be able to match it and not reject modify request due // to job not found - if (correct_jobname_p != NULL) + if (correct_jobname != preq->rq_ind.rq_modify.rq_objname) { - if (strcmp(correct_jobname_p, preq->rq_ind.rq_modify.rq_objname) != 0) - { - // job names don't match so need to modify the requested jobname - snprintf(preq->rq_ind.rq_modify.rq_objname, - sizeof(preq->rq_ind.rq_modify.rq_objname), - "%s", correct_jobname_p); - } - - free(correct_jobname_p); + // job names don't match so need to modify the requested jobname + snprintf(preq->rq_ind.rq_modify.rq_objname, + sizeof(preq->rq_ind.rq_modify.rq_objname), + "%s", correct_jobname.c_str()); } mutex_mgr job_mutex(pjob->ji_mutex, true); diff --git a/src/server/req_register.c b/src/server/req_register.c index a3f08966e0..26754c02aa 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -122,7 +122,7 @@ extern int que_to_local_svr(struct batch_request *preq); extern long calc_job_cost(job *); -char *get_correct_jobname(const char *jobid); +const char *get_correct_jobname(const char *jobid, std::string &corrected); /* Local Private Functions */ @@ -375,13 +375,12 @@ bool job_ids_match( if ((is_svr_attr_set(SRV_ATR_display_job_server_suffix)) || (is_svr_attr_set(SRV_ATR_job_suffix_alias))) { - char *correct_parent = get_correct_jobname(parent); - char *correct_child = get_correct_jobname(child); + std::string correct_parent; + std::string correct_child; + get_correct_jobname(parent, correct_parent); + get_correct_jobname(child, correct_child); - match = strcmp(correct_parent, correct_child) == 0; - - free(correct_parent); - free(correct_child); + match = correct_parent == correct_child; } else match = strcmp(parent, child) == 0; @@ -3221,13 +3220,16 @@ int build_depend( else /* all other dependency types */ { /* a set of job_id[\:port][@server[\:port]] */ + std::string correct_id; pdjb = new depend_job(); + get_correct_jobname(valwd, correct_id); + if (pdjb) { pdjb->dc_state = 0; pdjb->dc_cost = 0; - pdjb->dc_child = valwd; + pdjb->dc_child = correct_id; std::size_t pos = pdjb->dc_child.find("@"); diff --git a/src/test/array_func/scaffolding.c b/src/test/array_func/scaffolding.c index e06c06f6d4..9b32eeaeb4 100644 --- a/src/test/array_func/scaffolding.c +++ b/src/test/array_func/scaffolding.c @@ -133,9 +133,10 @@ void svr_evaljobstate(job &pjob, int &newstate, int &newsub, int forceeval) exit(1); } -char *get_correct_jobname(const char *jobid) +const char *get_correct_jobname(const char *jobid, std::string &correct) { - return(strdup(jobid)); + correct = jobid; + return(correct.c_str()); } void *get_prior(list_link pl, char *file, int line) diff --git a/src/test/job_container/test_uut.c b/src/test/job_container/test_uut.c index 0b09547874..96ddfba863 100644 --- a/src/test/job_container/test_uut.c +++ b/src/test/job_container/test_uut.c @@ -5,7 +5,7 @@ #include "pbs_error.h" #include -char *get_correct_jobname(const char *jobid); +const char *get_correct_jobname(const char *jobid, std::string &correct); void log_err(int,const char *,const char *) {} @@ -14,8 +14,9 @@ void log_err(int,const char *,const char *) START_TEST(get_correct_jobname_test) { // with nothing set, get_correct_jobname should just return the jobid passed in. - char *jobid = get_correct_jobname("1.napali.ac"); - fail_unless(!strcmp(jobid, "1.napali.ac")); + std::string correct; + get_correct_jobname("1.napali.ac", correct); + fail_unless(correct == "1.napali.ac"); } END_TEST diff --git a/src/test/req_modify/scaffolding.c b/src/test/req_modify/scaffolding.c index ac17cb4a9d..fe7a15080e 100644 --- a/src/test/req_modify/scaffolding.c +++ b/src/test/req_modify/scaffolding.c @@ -27,7 +27,7 @@ int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ static int acl_check_n = 0; -char *get_correct_jobname_return; +std::string get_correct_jobname_return; struct batch_request *alloc_br(int type) { @@ -270,12 +270,16 @@ batch_request *duplicate_request(batch_request *preq, int type) return(NULL); } -char *get_correct_jobname(char const *p) +const char *get_correct_jobname(const char *jobid, std::string &correct) { - if (get_correct_jobname_return == NULL) - return(strdup(p)); + if (get_correct_jobname_return.size() == 0) + { + correct = jobid; + } + else + correct = get_correct_jobname_return; - return(get_correct_jobname_return); + return(correct.c_str()); } job *chk_job_request(char *p, batch_request *b) diff --git a/src/test/req_modify/test_uut.c b/src/test/req_modify/test_uut.c index f57ee83ddd..9a229fbc2e 100644 --- a/src/test/req_modify/test_uut.c +++ b/src/test/req_modify/test_uut.c @@ -7,14 +7,14 @@ batch_request *preq; -extern char *get_correct_jobname_return; +extern std::string get_correct_jobname_return; bool allowed_gres_modifier(const char*, const char *); START_TEST(test_req_modifyjob) { // initialize - get_correct_jobname_return = NULL; + get_correct_jobname_return.clear(); // create space for the req preq = (batch_request *)calloc(1, sizeof(batch_request)); @@ -32,7 +32,7 @@ START_TEST(test_req_modifyjob) sizeof(preq->rq_ind.rq_modify.rq_objname))); // set up get_correct_jobname() to return a different value - get_correct_jobname_return = strdup("123"); + get_correct_jobname_return = "123"; // call the function to unit test req_modifyjob(preq); diff --git a/src/test/req_register/scaffolding.c b/src/test/req_register/scaffolding.c index 0fd27f9b0f..4b39c3adeb 100644 --- a/src/test/req_register/scaffolding.c +++ b/src/test/req_register/scaffolding.c @@ -357,7 +357,7 @@ job *job_alloc(void) return(pj); } -char *get_correct_jobname(const char *jobid) +const char *get_correct_jobname(const char *jobid, std::string &correct) { char *rv = strdup(jobid); @@ -366,7 +366,10 @@ char *get_correct_jobname(const char *jobid) if ((dot = strchr(rv, '.')) != NULL) *dot = '\0'; - return(rv); + correct = rv; + free(rv); + + return(correct.c_str()); } int is_svr_attr_set(int index) From 05098b9a6e0e065505696bea023f191595e295de Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 6 Dec 2016 11:29:55 -0700 Subject: [PATCH 088/312] TRQ-3861. Fix a unit test. --- src/test/numa_pci_device/test_uut.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/numa_pci_device/test_uut.c b/src/test/numa_pci_device/test_uut.c index f5acaeb3f8..95555c8a7f 100644 --- a/src/test/numa_pci_device/test_uut.c +++ b/src/test/numa_pci_device/test_uut.c @@ -8,12 +8,13 @@ START_TEST(test_displayAsString) { PCI_Device p; - p.setName("gpu"); + p.setName("k80"); + p.set_type(GPU); p.setId(0); std::stringstream out; p.displayAsString(out); - fail_unless(out.str() == " pci 0 gpu\n", out.str().c_str()); + fail_unless(out.str() == " GPU 0 k80\n", out.str().c_str()); } END_TEST From 0781f17925e29b33a3642faa8f21da48f9ab5951 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 6 Dec 2016 11:45:47 -0700 Subject: [PATCH 089/312] TRQ-3735. Fix another unit test. --- src/test/job_recov/scaffolding.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/job_recov/scaffolding.c b/src/test/job_recov/scaffolding.c index a6fe72134a..e2f57d65b8 100644 --- a/src/test/job_recov/scaffolding.c +++ b/src/test/job_recov/scaffolding.c @@ -323,9 +323,10 @@ const char *id_map::get_name(int internal_job_id) id_map job_mapper; -char *get_correct_jobname(const char *id) +const char *get_correct_jobname(const char *id, std::string &correct) { - return(strdup(id)); + correct = id; + return(correct.c_str()); } int encode_complete_req( From 024a2d3bfbfe2185c673c42de3d1410e5cfb7fd5 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 6 Dec 2016 15:52:44 -0700 Subject: [PATCH 090/312] TRQ-3862. Fix a command line vs. submit filter precedence error for interactive jobs. Interactive jobs were getting variables from the submit filter over command line variables. --- src/cmds/qsub_functions.c | 2 +- src/lib/Libcmds/add_verify_resources.c | 5 +++-- src/lib/Libutils/u_hash_map_structs.c | 17 ++++++++++++++++- src/test/u_hash_map_structs/test_uut.c | 15 +++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 2369bbe953..b3fbefd494 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -3816,7 +3816,7 @@ void process_opts( case 'l': - if (add_verify_resources(ji->res_attr, vptr, data_type)) + if (add_verify_resources(ji->res_attr, vptr, FILTER_DATA)) print_qsub_usage_exit("qsub: illegal -l value"); break; diff --git a/src/lib/Libcmds/add_verify_resources.c b/src/lib/Libcmds/add_verify_resources.c index 4ebc0851bf..e7f0da4dc8 100644 --- a/src/lib/Libcmds/add_verify_resources.c +++ b/src/lib/Libcmds/add_verify_resources.c @@ -109,9 +109,10 @@ /* hostlist=name1+name2,mppnodes=name1+name2,param=val1+val23+val24+val25*/ int add_verify_resources( + job_data_container *res_attr, /* M */ - char *resources, /* I */ - int p_type) /* I */ + char *resources, /* I */ + int p_type) /* I */ { char *r; diff --git a/src/lib/Libutils/u_hash_map_structs.c b/src/lib/Libutils/u_hash_map_structs.c index aac532f401..525c32a1f0 100644 --- a/src/lib/Libutils/u_hash_map_structs.c +++ b/src/lib/Libutils/u_hash_map_structs.c @@ -107,10 +107,25 @@ int hash_add_item( { job_data *item = new job_data(name,value,var_type,op_type); + bool replace = false; head->lock(); - int ret = head->insert(item,name,true); + job_data *old_item = head->find(name); + int ret = 1; + + if (old_item != NULL) + { + // Only call insert if we are replacing the old item + if (var_type < old_item->var_type) + ret = head->insert(item, name, true); + else + delete item; + } + else + ret = head->insert(item, name, replace); + head->unlock(); + return ret; } /* END hash_add_item() */ diff --git a/src/test/u_hash_map_structs/test_uut.c b/src/test/u_hash_map_structs/test_uut.c index 6acc507dbf..a33cf0d497 100644 --- a/src/test/u_hash_map_structs/test_uut.c +++ b/src/test/u_hash_map_structs/test_uut.c @@ -128,6 +128,8 @@ START_TEST(test_hash_print) rc = hash_clear(&the_map); } END_TEST + + /* Testing this involves forcing this to exit - causing a failure. Don't test. START_TEST(test_add_or_exit) { @@ -183,6 +185,19 @@ START_TEST(test_hash_add_hash) fail_unless(rc == 4, "4 items should exist in the hash_map, %d do", rc); fail_unless(hash_find(&map1, "name4", &tmp_item) == TRUE, "hash item name4 not found"); fail_unless(strcmp(tmp_item->value.c_str(), "value6") == 0, "value for name4 should have been value6, but was %s", tmp_item->value.c_str()); + + // Make sure we don't update the lower priority value + rc = hash_add_item(&map1, "name1", "bob", SCRIPT_DATA, SET); + fail_unless(rc == 1, "We should get success even though we didn't update lower priority data"); + job_data *stored; + fail_unless(hash_find(&map1, "name1", &stored) == 1); + fail_unless(stored->value == "value1"); + + fail_unless(hash_add_item(&map1, "planet", "sel", SCRIPT_DATA, SET) == 1); + fail_unless(hash_add_item(&map1, "planet", "roshar", ENV_DATA, SET) == 1); + // Make sure the value is now roshar - it is higher priority + fail_unless(hash_find(&map1, "planet", &stored) == 1); + fail_unless(stored->value == "roshar"); } END_TEST From a9b8ece258e923677023e343198105f776439ffa Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 6 Dec 2016 17:34:50 -0700 Subject: [PATCH 091/312] TRQ-3847 - change select() instances to poll() --- src/cmds/qsub_functions.c | 77 +++++---- src/include/net_connect.h | 1 - src/lib/Libifl/tcp_dis.c | 2 +- src/lib/Libnet/lib_net.h | 6 - src/lib/Libnet/net_client.c | 98 ++++------- src/lib/Libnet/net_common.c | 26 ++- src/lib/Libnet/net_server.c | 229 ++++++++++++++++---------- src/lib/Libnet/port_forwarding.c | 71 ++++---- src/lib/Libnet/rm.c | 70 ++++---- src/resmom/mom_comm.c | 83 +++++++--- src/resmom/mom_main.c | 6 +- src/resmom/pbs_demux.c | 82 ++++++--- src/resmom/start_exec.c | 23 +-- src/test/mail_throttler/scaffolding.c | 3 +- src/test/mom_comm/test_uut.c | 12 ++ src/test/mom_main/test_uut.c | 22 +-- src/test/net_client/scaffolding.c | 10 ++ src/test/net_client/test_uut.c | 24 ++- src/test/net_common/scaffolding.c | 23 +++ src/test/net_common/test_uut.c | 43 +++++ src/test/net_server/scaffolding.c | 37 +++-- src/test/net_server/test_uut.c | 28 ++++ src/test/pbs_demux/test_uut.c | 14 +- src/test/qsub_functions/scaffolding.c | 11 ++ src/test/qsub_functions/test_uut.c | 48 ++++++ src/test/rm/scaffolding.c | 16 +- src/test/rm/test_uut.c | 14 +- src/test/start_exec/scaffolding.c | 15 ++ src/test/start_exec/test_uut.c | 23 +++ 29 files changed, 746 insertions(+), 371 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index b3fbefd494..a6f2fbe4c5 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -38,6 +38,7 @@ #include #include #include +#include #ifdef sun #include @@ -47,10 +48,6 @@ #include #endif -#if defined(FD_SET_IN_SYS_SELECT_H) -#include -#endif - #include "libcmds.h" /* TShowAbout_exit */ #include "cmds.h" #include "net_connect.h" @@ -2169,6 +2166,48 @@ void x11handler( exit(EXIT_FAILURE); } +/* + * wait_for_read_ready() + * + * Wait for a file descriptor to be ready to read before timeout elapsed + * @param fd - file descriptor + * @param timeout_sec - time in seconds to wait for read activity + * @return -1 if error, 0 if not ready, >1 if ready + */ + +int wait_for_read_ready( + + int fd, + int timeout_sec) + + { + struct pollfd PollArray; + int n; + + PollArray.fd = fd; + PollArray.events = POLLIN; + PollArray.revents = 0; + + // wait for data ready to read + n = poll(&PollArray, 1, timeout_sec * 1000); + + if (n > 0) + { + // events returned, fd ready for reading? + if ((PollArray.revents & POLLIN) == 0) + { + // none ready to read after timeout + n = 0; + } + } + else if ((n == -1) && (errno == EINTR)) + { + // ignore signals -- none ready to read + n = 0; + } + + return(n); + } /* @@ -2184,17 +2223,13 @@ void interactive( char momjobid[LOG_BUF_SIZE+1]; int news; - int nsel; char *pc; - fd_set selset; struct sigaction act; struct sockaddr_in from; torque_socklen_t fromlen; - struct timeval timeout; - struct winsize wsz; job_data *tmp_job_info; @@ -2236,31 +2271,15 @@ void interactive( /* Accept connection on socket set up earlier */ - nsel = 0; - - while (nsel == 0) + while (true) { - FD_ZERO(&selset); - FD_SET(inter_sock, &selset); + int rc; - timeout.tv_usec = 0; - timeout.tv_sec = 30; + if ((rc = wait_for_read_ready(inter_sock, 30)) < 0) + print_qsub_usage_exit("qsub: poll failed"); - nsel = select(FD_SETSIZE, &selset, NULL, NULL, &timeout); - - if (nsel > 0) - { + if (rc > 0) break; - } - else if (nsel == -1) - { - if (errno == EINTR) - { - nsel = 0; - } - else - print_qsub_usage_exit("qsub: select failed"); - } /* connect to server, status job to see if still there */ diff --git a/src/include/net_connect.h b/src/include/net_connect.h index 1555f351a0..7638ad7c6d 100644 --- a/src/include/net_connect.h +++ b/src/include/net_connect.h @@ -213,7 +213,6 @@ void net_close(int); int wait_request(time_t waittime, long *); void net_add_close_func(int, void(*func)(int)); int get_max_num_descriptors(void); -int get_fdset_size(void); char * netaddr_pbs_net_t(pbs_net_t); #ifdef __cplusplus } diff --git a/src/lib/Libifl/tcp_dis.c b/src/lib/Libifl/tcp_dis.c index bb1e880be1..35b072a600 100644 --- a/src/lib/Libifl/tcp_dis.c +++ b/src/lib/Libifl/tcp_dis.c @@ -107,7 +107,7 @@ #endif #define MAX_SOCKETS 65536 -time_t pbs_tcp_timeout = 300; +time_t pbs_tcp_timeout = 300; // seconds diff --git a/src/lib/Libnet/lib_net.h b/src/lib/Libnet/lib_net.h index 85b789b2c0..6b5929c7ea 100644 --- a/src/lib/Libnet/lib_net.h +++ b/src/lib/Libnet/lib_net.h @@ -64,15 +64,10 @@ int start_listener_addrinfo(char *host_name, int server_port, void *(*process_me int bindresvport(int sd, struct sockaddr_in *sin); #endif int get_max_num_descriptors(void); -int get_fdset_size(void); /* static int await_connect(long timeout, int sockd); */ int client_to_svr(pbs_net_t hostaddr, unsigned int port, int local_port, char *EMsg); /* from file net_server.c */ -void global_sock_add(int new_sock); -void global_sock_rem(int new_sock); -fd_set *global_sock_getlist(); -int global_sock_verify(); void netcounter_incr(void); int get_num_connections(); void netcounter_get(int netrates[]); @@ -80,7 +75,6 @@ int init_network(unsigned int port, void *(*readfunc)(void *)); int check_network_port(unsigned int port); int check_trqauthd_unix_domain_port(const char *); int ping_trqauthd(const char *); -int thread_func(int active_sockets, fd_set *select_set); int wait_request(time_t waittime, long *SState); /* static void accept_conn(void *new_conn); */ void globalset_add_sock(int sock); diff --git a/src/lib/Libnet/net_client.c b/src/lib/Libnet/net_client.c index ad220e6e6f..0dff89d684 100644 --- a/src/lib/Libnet/net_client.c +++ b/src/lib/Libnet/net_client.c @@ -95,6 +95,7 @@ #include #include #include +#include #ifdef _AIX #include @@ -124,48 +125,41 @@ int get_max_num_descriptors(void) return(max_num_descriptors); } /* END get_num_max_descriptors() */ -/** - * Returns the number of bytes needed to allocate - * a fd_set array that can hold all of the possible - * socket descriptors. +/* + * wait_for_write_ready + * + * Wait for file descriptor to be ready for writing + * within timeout period. + * + * @param fd - file descriptor + * @param timeout_ms - timeout in milliseconds + * @return true if file descriptor ready for writing, false otherwise */ -int get_fdset_size(void) - - { - unsigned int MaxNumDescriptors = 0; - int NumFDSetsNeeded = 0; - int NumBytesInFDSet = 0; - int Result = 0; - - MaxNumDescriptors = get_max_num_descriptors(); +bool wait_for_write_ready( - NumBytesInFDSet = sizeof(fd_set); - NumFDSetsNeeded = MaxNumDescriptors / FD_SETSIZE; + int fd, + int timeout_ms) - if (MaxNumDescriptors < FD_SETSIZE) - { - /* the default size already provides sufficient space */ + { + struct pollfd PollArray; - Result = NumBytesInFDSet; - } - else if ((MaxNumDescriptors % FD_SETSIZE) > 0) - { - /* we need to allocate more memory to cover extra - * bits--add an extra FDSet worth of memory to the size */ + // initialize + PollArray.fd = fd; + PollArray.events = POLLOUT; + PollArray.revents = 0; - Result = (NumFDSetsNeeded + 1) * NumBytesInFDSet; - } - else + // wait + if ((poll(&PollArray, 1, timeout_ms) == 1) && + ((PollArray.revents & POLLOUT) != 0)) { - /* division was exact--we know exactly how many bytes we need */ - - Result = NumFDSetsNeeded * NumBytesInFDSet; + // ready to write + return(true); } - return(Result); - } /* END get_fdset_size() */ - + // not ready to write + return(false); + } /* ** wait for connect to complete. We use non-blocking sockets, @@ -174,45 +168,19 @@ int get_fdset_size(void) static int await_connect( - long timeout, /* I */ + int timeout_ms, /* I - milliseconds */ int sockd) /* I */ { - int n; int val; int rc; - fd_set *BigFDSet = NULL; - - struct timeval tv; - torque_socklen_t len; - /* - * some operating systems (like FreeBSD) cannot have a value for tv.tv_usec - * larger than 1,000,000 so we need to split up the timeout duration between - * seconds and microseconds - */ - - tv.tv_sec = timeout / 1000000; - tv.tv_usec = timeout % 1000000; - - /* calculate needed size for fd_set in select() */ - - BigFDSet = (fd_set *)calloc(1,sizeof(char) * get_fdset_size()); - if (!BigFDSet) - { - log_err(ENOMEM,__func__,"Could not allocate memory to set file descriptor"); - return -1; - } - - FD_SET(sockd, BigFDSet); - - if ((n = select(sockd+1,0,BigFDSet,0,&tv)) != 1) + if (wait_for_write_ready(sockd, timeout_ms) == false) { /* FAILURE: socket not ready for write */ - free(BigFDSet); return(-1); } @@ -224,7 +192,6 @@ static int await_connect( { /* SUCCESS: no failures detected */ - free(BigFDSet); return(0); } @@ -232,7 +199,6 @@ static int await_connect( /* FAILURE: socket error detected */ - free(BigFDSet); return(-1); } /* END await_connect() */ @@ -240,7 +206,7 @@ static int await_connect( /* global */ -long MaxConnectTimeout = 5000000; /* in microseconds */ +int MaxConnectTimeout = 5000; /* in milliseconds */ /* * client_to_svr - connect to a server @@ -257,7 +223,7 @@ long MaxConnectTimeout = 5000000; /* in microseconds */ * hosts with the same port. Let the caller keep the addresses around * rather than look it up each time. * - * NOTE: will wait up to MaxConnectTimeout microseconds for transient network failures + * NOTE: will wait up to MaxConnectTimeout milliseconds for transient network failures */ /* NOTE: create new connection on reserved port to validate root/trusted authority */ @@ -562,7 +528,7 @@ int client_to_svr( if (await_connect(MaxConnectTimeout, sock) == 0) { - /* socket not ready for writing after MaxConnectTimeout microseconds timeout */ + /* socket not ready for writing after MaxConnectTimeout milliseconds timeout */ /* no network failures detected */ break; diff --git a/src/lib/Libnet/net_common.c b/src/lib/Libnet/net_common.c index 2e1bfe7e6c..b78e2c6331 100644 --- a/src/lib/Libnet/net_common.c +++ b/src/lib/Libnet/net_common.c @@ -500,7 +500,7 @@ int process_and_save_socket_error( * socket_wait_for_write() * * connect failed, this function determines why. For non-blocking sockets, - * EINPROGRESS is almost always set, and you need to select the socket and then + * EINPROGRESS is almost always set, and you need to poll the socket and then * read error using getsockopt(), as is done below. * * if the failure is a permanent failure, pass that back to the caller @@ -520,25 +520,23 @@ int socket_wait_for_write( int write_soc = 0; int sock_errno = 0; socklen_t len = sizeof(int); - fd_set wfd; - struct timeval timeout; + struct pollfd PollArray; - timeout.tv_sec = pbs_tcp_timeout; - timeout.tv_usec = 0; + PollArray.fd = socket; + PollArray.events = POLLOUT; + PollArray.revents = 0; - FD_ZERO(&wfd); - FD_SET(socket, &wfd); - - if ((write_soc = select(socket+1, 0, &wfd, 0, &timeout)) != 1) + if ((write_soc = poll(&PollArray, 1, pbs_tcp_timeout * 1000)) != 1) { /* timeout is now seen as a permanent failure */ rc = PERMANENT_SOCKET_FAIL; } - else if (((rc = getsockopt(socket, SOL_SOCKET, SO_ERROR, &sock_errno, &len)) == 0) && + else if ((PollArray.revents & POLLOUT) && + ((rc = getsockopt(socket, SOL_SOCKET, SO_ERROR, &sock_errno, &len)) == 0) && (sock_errno == 0)) - { - rc = PBSE_NONE; - } + { + rc = PBSE_NONE; + } else { rc = process_and_save_socket_error(sock_errno); @@ -579,7 +577,7 @@ int socket_wait_for_xbytes( int socket_wait_for_read( int socket, - unsigned int timeout) + unsigned int timeout) // seconds { int rc = PBSE_NONE; diff --git a/src/lib/Libnet/net_server.c b/src/lib/Libnet/net_server.c index f1a3f8e344..618e139058 100644 --- a/src/lib/Libnet/net_server.c +++ b/src/lib/Libnet/net_server.c @@ -102,9 +102,7 @@ #include #include /* thread_func functions */ -#if defined(FD_SET_IN_SYS_SELECT_H) -#include -#endif +#include #if defined(NTOHL_NEEDS_ARPA_INET_H) && defined(HAVE_ARPA_INET_H) #include #endif @@ -153,7 +151,7 @@ struct connection svr_conn[PBS_NET_MAX_CONNECTIONS]; static int max_connection = PBS_NET_MAX_CONNECTIONS; static int num_connections = 0; pthread_mutex_t *num_connections_mutex = NULL; -static fd_set *GlobalSocketReadSet = NULL; +static struct pollfd *GlobalSocketReadArray = NULL; static u_long *GlobalSocketAddrSet = NULL; static u_long *GlobalSocketPortSet = NULL; pthread_mutex_t *global_sock_read_mutex = NULL; @@ -263,7 +261,7 @@ void netcounter_get( /** * init_network - initialize the network interface * allocate a socket and bind it to the service port, - * add the socket to the readset for select(), + * add the socket to the read set for poll(), * add the socket to the connection structure and set the * processing function to accept_conn() */ @@ -323,7 +321,7 @@ int init_network( disiui_(); - for (i = 0;i < PBS_NET_MAX_CONNECTIONS;i++) + for (i = 0; i < PBS_NET_MAX_CONNECTIONS; i++) { svr_conn[i].cn_mutex = (pthread_mutex_t *)calloc(1, sizeof(pthread_mutex_t)); pthread_mutex_init(svr_conn[i].cn_mutex,NULL); @@ -334,10 +332,40 @@ int init_network( pthread_mutex_unlock(svr_conn[i].cn_mutex); } - /* initialize global "read" socket FD bitmap */ - GlobalSocketReadSet = (fd_set *)calloc(1,sizeof(char) * get_fdset_size()); - GlobalSocketAddrSet = (u_long *)calloc(sizeof(ulong),get_max_num_descriptors()); - GlobalSocketPortSet = (u_long *)calloc(sizeof(ulong),get_max_num_descriptors()); + /* initialize global "read" socket array */ + MaxNumDescriptors = get_max_num_descriptors(); + + GlobalSocketReadArray = (struct pollfd *)malloc(MaxNumDescriptors * sizeof(struct pollfd)); + if (GlobalSocketReadArray == NULL) + { + return(-1); // no memory + } + + for (i = 0; i < MaxNumDescriptors; i++) + { + GlobalSocketReadArray[i].fd = -1; + GlobalSocketReadArray[i].events = 0; + GlobalSocketReadArray[i].revents = 0; + } + + GlobalSocketAddrSet = (u_long *)calloc(MaxNumDescriptors, sizeof(ulong)); + if (GlobalSocketAddrSet == NULL) + { + free(GlobalSocketReadArray); + GlobalSocketReadArray = NULL; + return(-1); // no memory + } + + GlobalSocketPortSet = (u_long *)calloc(MaxNumDescriptors, sizeof(ulong)); + if (GlobalSocketPortSet == NULL) + { + free(GlobalSocketReadArray); + GlobalSocketReadArray = NULL; + + free(GlobalSocketAddrSet); + GlobalSocketAddrSet = NULL; + return(-1); // no memory + } global_sock_read_mutex = (pthread_mutex_t *)calloc(1, sizeof(pthread_mutex_t)); pthread_mutex_init(global_sock_read_mutex,&t_attr); @@ -398,7 +426,7 @@ int init_network( return(-1); } - /* record socket in connection structure and select set */ + /* record socket in connection structure and poll set */ add_conn(sock, type, (pbs_net_t)0, 0, PBS_SOCK_INET, accept_conn); @@ -647,7 +675,7 @@ int check_network_port( /* * wait_request - wait for a request (socket with data to read) - * This routine does a select on the readset of sockets, + * This routine does a poll on the read set of sockets, * when data is ready, the processing routine associated with * the socket is invoked. */ @@ -662,58 +690,70 @@ int wait_request( int n; time_t now; - fd_set *SelectSet = NULL; - int SelectSetSize = 0; + struct pollfd *PollArray = NULL; + int PollArraySize = 0; int MaxNumDescriptors = 0; int SetSize = 0; - u_long *SocketAddrSet = NULL; - u_long *SocketPortSet = NULL; + u_long *SocketAddrSet = NULL; + u_long *SocketPortSet = NULL; char tmpLine[1024]; - struct timeval timeout; + int timeout_ms; long OrigState = 0; if (SState != NULL) OrigState = *SState; - timeout.tv_usec = 0; - timeout.tv_sec = waittime; + // scale seconds to milliseconds + timeout_ms = waittime * 1000; + + MaxNumDescriptors = get_max_num_descriptors(); - SelectSetSize = sizeof(char) * get_fdset_size(); - SelectSet = (fd_set *)calloc(1,SelectSetSize); + PollArraySize = MaxNumDescriptors * sizeof(struct pollfd); + PollArray = (struct pollfd *)malloc(PollArraySize); - if(SelectSet == NULL) + if (PollArray == NULL) { return(-1); } + + // since we don't want to hold the mutex for too long, + // just make a local copy of the data and act on it + pthread_mutex_lock(global_sock_read_mutex); - - memcpy(SelectSet,GlobalSocketReadSet,SelectSetSize); + + // copy global array to local one + memcpy(PollArray, GlobalSocketReadArray, PollArraySize); - /* selset = readset;*/ /* readset is global */ - MaxNumDescriptors = get_max_num_descriptors(); - SetSize = MaxNumDescriptors*sizeof(u_long); + SetSize = MaxNumDescriptors * sizeof(u_long); SocketAddrSet = (u_long *)malloc(SetSize); SocketPortSet = (u_long *)malloc(SetSize); - if((SocketAddrSet == NULL) || (SocketPortSet == NULL)) + + if ((SocketAddrSet == NULL) || (SocketPortSet == NULL)) { - free(SelectSet); - if(SocketAddrSet != NULL) free(SocketAddrSet); - if(SocketPortSet != NULL) free(SocketPortSet); + free(PollArray); + + if (SocketAddrSet != NULL) + free(SocketAddrSet); + if (SocketPortSet != NULL) + free(SocketPortSet); + pthread_mutex_unlock(global_sock_read_mutex); return (-1); } - memcpy(SocketAddrSet,GlobalSocketAddrSet,SetSize); - memcpy(SocketPortSet,GlobalSocketPortSet,SetSize); + + memcpy(SocketAddrSet, GlobalSocketAddrSet, SetSize); + memcpy(SocketPortSet, GlobalSocketPortSet, SetSize); pthread_mutex_unlock(global_sock_read_mutex); - n = select(MaxNumDescriptors, SelectSet, (fd_set *)0, (fd_set *)0, &timeout); + + n = poll(PollArray, MaxNumDescriptors, timeout_ms); if (n == -1) { - if (errno == EINTR) + if (errno == EINTR) { n = 0; /* interrupted, cycle around */ } @@ -725,26 +765,29 @@ int wait_request( /* check all file descriptors to verify they are valid */ - /* NOTE: selset may be modified by failed select() */ - for (i = 0; i < MaxNumDescriptors; i++) { - if (FD_ISSET(i, SelectSet) == 0) + // skip this entry, fd not set + if (PollArray[i].fd < 0) continue; - if (fstat(i, &fbuf) == 0) + // skip this entry, return events present + if (PollArray[i].revents != 0) continue; - /* clean up SdList and bad sd... */ + // skip this entry, it's a valid descriptor + if (fstat(i, &fbuf) == 0) + continue; + // remove socket from the global set since it's no longer valid globalset_del_sock(i); - } /* END for each socket in global read set */ + } /* END for each socket in global read array */ - free(SelectSet); + free(PollArray); free(SocketAddrSet); free(SocketPortSet); - log_err(errno, __func__, "Unable to select sockets to read requests"); + log_err(errno, __func__, "Unable to poll sockets to read requests"); return(-1); } /* END else (errno == EINTR) */ @@ -752,58 +795,65 @@ int wait_request( for (i = 0; (i < max_connection) && (n > 0); i++) { - if (FD_ISSET(i, SelectSet)) + // any events? + if (PollArray[i].revents != 0) { - pthread_mutex_lock(svr_conn[i].cn_mutex); - /* this socket has data */ + // decrement the count of structures that have non-zero revents n--; - svr_conn[i].cn_lasttime = time(NULL); - - if (svr_conn[i].cn_active != Idle) + // is there data ready to read? + if ((PollArray[i].revents & POLLIN)) { - void *(*func)(void *) = svr_conn[i].cn_func; + pthread_mutex_lock(svr_conn[i].cn_mutex); + /* this socket has data */ - netcounter_incr(); + svr_conn[i].cn_lasttime = time(NULL); - pthread_mutex_unlock(svr_conn[i].cn_mutex); - - if (func != NULL) + if (svr_conn[i].cn_active != Idle) { - int args[3]; + void *(*func)(void *) = svr_conn[i].cn_func; - args[0] = i; - args[1] = (int)SocketAddrSet[i]; - args[2] = (int)SocketPortSet[i]; - func((void *)args); - } + netcounter_incr(); - /* NOTE: breakout if state changed (probably received shutdown request) */ + pthread_mutex_unlock(svr_conn[i].cn_mutex); - if ((SState != NULL) && - (OrigState != *SState)) - break; - } - else - { - pthread_mutex_unlock(svr_conn[i].cn_mutex); + if (func != NULL) + { + int args[3]; - globalset_del_sock(i); - close_conn(i, FALSE); + args[0] = i; + args[1] = (int)SocketAddrSet[i]; + args[2] = (int)SocketPortSet[i]; + func((void *)args); + } - pthread_mutex_lock(num_connections_mutex); + /* NOTE: breakout if state changed (probably received shutdown request) */ - sprintf(tmpLine, "closed connections to fd %d - num_connections=%d (select bad socket)", - i, - num_connections); + if ((SState != NULL) && + (OrigState != *SState)) + break; + } + else + { + pthread_mutex_unlock(svr_conn[i].cn_mutex); + + globalset_del_sock(i); + close_conn(i, FALSE); + + pthread_mutex_lock(num_connections_mutex); - pthread_mutex_unlock(num_connections_mutex); - log_err(-1, __func__, tmpLine); + sprintf(tmpLine, "closed connections to fd %d - num_connections=%d (poll bad socket)", + i, + num_connections); + + pthread_mutex_unlock(num_connections_mutex); + log_err(-1, __func__, tmpLine); + } } } } /* END for i */ - free(SelectSet); + free(PollArray); free(SocketAddrSet); free(SocketPortSet); @@ -816,7 +866,7 @@ int wait_request( now = time((time_t *)0); - for (i = 0;i < max_connection;i++) + for (i = 0; i < max_connection; i++) { struct connection *cp; @@ -878,7 +928,7 @@ int wait_request( * accept_conn - accept request for new connection * this routine is normally associated with the main socket, * requests for connection on the socket are accepted and - * the new socket is added to the select set and the connection + * the new socket is added to the poll set and the connection * structure - the processing routine is set to the external * function: process_request(socket) * @@ -937,7 +987,7 @@ void *accept_conn( } else { - /* add the new socket to the select set and connection structure */ + /* add the new socket to the poll set and connection structure */ add_conn( newsock, FromClientDIS, @@ -962,7 +1012,8 @@ void globalset_add_sock( { pthread_mutex_lock(global_sock_read_mutex); - FD_SET(sock, GlobalSocketReadSet); + GlobalSocketReadArray[sock].fd = sock; + GlobalSocketReadArray[sock].events = POLLIN; GlobalSocketAddrSet[sock] = addr; GlobalSocketPortSet[sock] = port; pthread_mutex_unlock(global_sock_read_mutex); @@ -977,7 +1028,9 @@ void globalset_del_sock( { pthread_mutex_lock(global_sock_read_mutex); - FD_CLR(sock, GlobalSocketReadSet); + GlobalSocketReadArray[sock].fd = -1; + GlobalSocketReadArray[sock].events = 0; + GlobalSocketReadArray[sock].revents = 0; GlobalSocketAddrSet[sock] = 0; GlobalSocketPortSet[sock] = 0; pthread_mutex_unlock(global_sock_read_mutex); @@ -1000,7 +1053,7 @@ int add_connection( unsigned int port, /* port number (host order) on connected host */ unsigned int socktype, /* inet or unix */ void *(*func)(void *), /* function to invoke on data rdy to read */ - int add_wait_request) /* True to add into global select set */ + int add_wait_request) /* True to add into global poll set */ { if ((sock < 0) || @@ -1182,10 +1235,10 @@ void close_conn( /* * In the case of a -t cold start, this will be called prior to - * GlobalSocketReadSet being initialized + * GlobalSocketReadArray being initialized */ - if (GlobalSocketReadSet != NULL) + if (GlobalSocketReadArray != NULL) { globalset_del_sock(sd); } @@ -1252,10 +1305,10 @@ void clear_conn( /* * In the case of a -t cold start, this will be called prior to - * GlobalSocketReadSet being initialized + * GlobalSocketReadArray being initialized */ - if (GlobalSocketReadSet != NULL) + if (GlobalSocketReadArray != NULL) { globalset_del_sock(sd); } @@ -1298,7 +1351,7 @@ void net_close( { int i; - for (i = 0;i < max_connection;i++) + for (i = 0; i < max_connection; i++) { if (i != but) { diff --git a/src/lib/Libnet/port_forwarding.c b/src/lib/Libnet/port_forwarding.c index 1a144df346..0392d7b621 100644 --- a/src/lib/Libnet/port_forwarding.c +++ b/src/lib/Libnet/port_forwarding.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "lib_ifl.h" @@ -39,8 +40,9 @@ void port_forwarder( char *EMsg) /* O */ { - fd_set rfdset, wfdset, efdset; - int rc, maxsock = 0; + struct pollfd *PollArray; + int rc; + int i; struct sockaddr_in from; torque_socklen_t fromlen; @@ -48,12 +50,22 @@ void port_forwarder( fromlen = sizeof(from); + PollArray = (struct pollfd *)malloc(NUM_SOCKS * sizeof(struct pollfd)); + + if (PollArray == NULL) + { + return; // no memory + } + while (1) { - FD_ZERO(&rfdset); - FD_ZERO(&wfdset); - FD_ZERO(&efdset); - maxsock = 0; + // clear the array + for (i = 0; i < NUM_SOCKS; i++) + { + PollArray[i].fd = -1; + PollArray[i].events = 0; + PollArray[i].revents = 0; + } for (n = 0; n < NUM_SOCKS; n++) { @@ -62,42 +74,46 @@ void port_forwarder( if ((socks + n)->listening) { - FD_SET((socks + n)->sock, &rfdset); + PollArray[n].fd = (socks + n)->sock; + PollArray[n].events = POLLIN; + PollArray[n].revents = 0; } else { if ((socks + n)->bufavail < BUF_SIZE) - FD_SET((socks + n)->sock, &rfdset); + { + PollArray[n].fd = (socks + n)->sock; + PollArray[n].events = POLLIN; + PollArray[n].revents = 0; + } if ((socks + ((socks + n)->peer))->bufavail - (socks + ((socks + n)->peer))->bufwritten > 0) - FD_SET((socks + n)->sock, &wfdset); - - /*FD_SET((socks+n)->sock,&efdset);*/ + { + PollArray[n].fd = (socks + n)->sock; + PollArray[n].events |= POLLOUT; + PollArray[n].revents = 0; + } } - - maxsock = (socks + n)->sock > maxsock ? (socks + n)->sock : maxsock; } - maxsock++; - - rc = select(maxsock, &rfdset, &wfdset, &efdset, NULL); + rc = poll(PollArray, NUM_SOCKS, -1); if ((rc == -1) && (errno == EINTR)) continue; if (rc < 0) { - perror("port forwarding select()"); + perror("port forwarding poll()"); exit(EXIT_FAILURE); } - for (n = 0;n < NUM_SOCKS;n++) + for (n = 0; n < NUM_SOCKS; n++) { if (!(socks + n)->active) continue; - if (FD_ISSET((socks + n)->sock, &rfdset)) + if ((PollArray[n].revents & POLLIN)) { if ((socks + n)->listening) { @@ -160,15 +176,9 @@ void port_forwarder( (socks + n)->bufavail += rc; } } - } /* END if rfdset */ - } /* END foreach fd */ + } /* END if read */ - for (n = 0; n < NUM_SOCKS; n++) - { - if (!(socks + n)->active) - continue; - - if (FD_ISSET((socks + n)->sock, &wfdset)) + if ((PollArray[n].revents & POLLOUT)) { int peer = (socks + n)->peer; @@ -187,8 +197,7 @@ void port_forwarder( { (socks + peer)->bufwritten += rc; } - } /* END if wfdset */ - + } /* END if write */ } /* END foreach fd */ for (n2 = 0; n2 <= 1;n2++) @@ -217,6 +226,10 @@ void port_forwarder( } } } /* END while(1) */ + + // never executed but should satisfy code profilers looking + // for a malloc/free pairing + free(PollArray); } /* END port_forwarder() */ diff --git a/src/lib/Libnet/rm.c b/src/lib/Libnet/rm.c index a7dcfbc951..3ad0b2afd7 100644 --- a/src/lib/Libnet/rm.c +++ b/src/lib/Libnet/rm.c @@ -96,6 +96,7 @@ #include #include #include +#include #include "pbs_ifl.h" #include "pbs_error.h" @@ -1048,70 +1049,65 @@ int activereq(void) #endif int i, num; + int PollArraySize; + int PollArrayIndex = 0; - struct timeval tv; - - fd_set *FDSet; - - int MaxNumDescriptors = 0; + struct pollfd *PollArray; flushreq(); - MaxNumDescriptors = get_max_num_descriptors(); - FDSet = (fd_set *)calloc(1,sizeof(char) * get_fdset_size()); + // initialize the poll array + PollArraySize = get_max_num_descriptors(); + PollArray = (struct pollfd *)calloc(PollArraySize, sizeof(struct pollfd)); - for (i = 0; i < HASHOUT; i++) + if (PollArray == NULL) { + // no memory + DBPRT(("%s: calloc %d %s\n", id, errno, pbs_strerror(errno))) + return(-1); + } + for (i = 0; (i < HASHOUT) && (PollArrayIndex < PollArraySize); i++) + { struct out *op; - op = outs[i]; - - while (op) + for (op = outs[i]; (op != NULL) && (PollArrayIndex < PollArraySize); op = op->next) { - FD_SET(op->chan->sock, FDSet); - op = op->next; + PollArray[PollArrayIndex].fd = op->chan->sock; + PollArray[PollArrayIndex].events = POLLIN; + PollArray[PollArrayIndex].revents = 0; + PollArrayIndex++; } } - tv.tv_sec = 15; - - tv.tv_usec = 0; - - num = select(MaxNumDescriptors, FDSet, NULL, NULL, &tv); + // poll with 15sec timeout + num = poll(PollArray, PollArrayIndex, 15000); if (num == -1) { - DBPRT(("%s: select %d %s\n", id, errno, pbs_strerror(errno))) - free(FDSet); + DBPRT(("%s: poll %d %s\n", id, errno, pbs_strerror(errno))) + free(PollArray); return -1; } else if (num == 0) { - free(FDSet); - return -2; + free(PollArray); + return -2; } - for (i = 0; i < HASHOUT; i++) + for (i = 0; i < PollArrayIndex; i++) { - - struct out *op; - - op = outs[i]; - - while (op) + // something to read? + if ((PollArray[i].revents & POLLIN)) { - if (FD_ISSET(op->chan->sock, FDSet)) - { - free(FDSet); - return op->chan->sock; - } - - op = op->next; + // return socket id that is ready for reading + int sock = PollArray[i].fd; + free(PollArray); + return(sock); } } - free(FDSet); + free(PollArray); return(-2); } /* END activereq() */ diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index 055a5a83c9..f1aa0c9cd4 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -99,6 +99,7 @@ #include #endif #include +#include #include "libpbs.h" #include "list_link.h" @@ -209,7 +210,7 @@ struct routefd unsigned short r_fd; }; -fd_set readset; +struct pollfd *readset = NULL; /* external functions */ @@ -8848,6 +8849,10 @@ int readit( char buf[READ_BUF_SIZE]; size_t ret; + // confirm readset has been initialized + if (readset == NULL) + return(-2); + if ((amt = recv(sock, buf, READ_BUF_SIZE, 0)) > 0) { ret = send(fd, buf, amt, 0); @@ -8855,14 +8860,21 @@ int readit( { close(sock); close(fd); - FD_CLR(sock, &readset); + + // remove sock from readset + readset[sock].fd = -1; + readset[sock].events = 0; + readset[sock].revents = 0; } } else { close(sock); - FD_CLR(sock, &readset); + // remove sock from readset + readset[sock].fd = -1; + readset[sock].events = 0; + readset[sock].revents = 0; } return(amt); @@ -8887,7 +8899,6 @@ void fork_demux( { pid_t cpid; - struct timeval timeout; int i; int retries; int maxfd; @@ -8897,7 +8908,6 @@ void fork_demux( int fd2; int im_mom_stdout; int im_mom_stderr; - fd_set selset; pid_t parent; u_long ipaddr; struct sigaction act; @@ -8907,6 +8917,8 @@ void fork_demux( int pipes[2]; int pipe_failed = FALSE; char buf[MAXLINE]; + int pollset_size_bytes; + struct pollfd *pollset; if ((maxfd = sysconf(_SC_OPEN_MAX)) < 0) { @@ -9003,9 +9015,37 @@ void fork_demux( /* maxfd = sysconf(_SC_OPEN_MAX); */ - FD_ZERO(&readset); - FD_SET(im_mom_stdout, &readset); - FD_SET(im_mom_stderr, &readset); + pollset_size_bytes = maxfd * sizeof(struct pollfd); + + readset = (struct pollfd *)malloc(pollset_size_bytes); + if (readset == NULL) + { + perror("failed to malloc memory for readset"); + _exit(5); + } + + // initialize readset + for (i = 0; i < maxfd; i++) + { + readset[i].fd = -1; + readset[i].events = 0; + readset[i].revents = 0; + } + + readset[im_mom_stdout].fd = im_mom_stdout; + readset[im_mom_stdout].events = POLLIN; + readset[im_mom_stdout].revents = 0; + + readset[im_mom_stderr].fd = im_mom_stderr; + readset[im_mom_stderr].events = POLLIN; + readset[im_mom_stderr].revents = 0; + + pollset = (struct pollfd *)malloc(pollset_size_bytes); + if (pollset == NULL) + { + perror("failed to malloc memory for pollset"); + _exit(5); + } if (listen(im_mom_stdout, TORQUE_LISTENQUEUE) < 0) { @@ -9086,11 +9126,11 @@ void fork_demux( while (1) { - selset = readset; - timeout.tv_usec = 0; - timeout.tv_sec = 20; - - n = select(FD_SETSIZE, &selset, (fd_set *)0, (fd_set *)0, &timeout); + // copy readset to local set for poll + memcpy(pollset, readset, pollset_size_bytes); + + // wait for up to 20sec + n = poll(pollset, maxfd, 20000); if (n == -1) { @@ -9100,7 +9140,7 @@ void fork_demux( } else { - perror("fork_demux: select failed\n"); + perror("fork_demux: poll failed\n"); close(im_mom_stdout); close(im_mom_stderr); close(fd1); @@ -9123,12 +9163,14 @@ void fork_demux( } /* END else if (n == 0) */ - for (i = 0;(n != 0) && (i < maxfd);++i) + for (i = 0; (n > 0) && (i < maxfd); i++) { - if (FD_ISSET(i, &selset)) + // decrement count of structures that have return events + n--; + + if ((pollset[i].revents & POLLIN)) { /* this socket has data */ - n--; switch (routem[i].r_which) { @@ -9151,8 +9193,11 @@ void fork_demux( routem[newsock].r_which = routem[i].r_which == listen_out ? new_out : new_err; routem[newsock].r_fd = newsock; open_sockets++; - - FD_SET(newsock, &readset); + + // add new socket to readset for future polling + readset[newsock].fd = newsock; + readset[newsock].events = POLLIN; + readset[newsock].revents = 0; break; diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index 9f7648d5b8..0e82028a69 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -6473,7 +6473,7 @@ void prepare_child_tasks_for_delete() -time_t calculate_select_timeout() { +time_t calculate_poll_timeout() { time_t tmpTime; extern time_t wait_time; @@ -6615,11 +6615,11 @@ void main_loop(void) time_now = time((time_t *)0); - tmpTime = calculate_select_timeout(); + tmpTime = calculate_poll_timeout(); resend_things(); - /* wait_request does a select and then calls the connection's cn_func for sockets with data */ + /* wait_request does a poll and then calls the connection's cn_func for sockets with data */ if (wait_request(tmpTime, NULL) != 0) { diff --git a/src/resmom/pbs_demux.c b/src/resmom/pbs_demux.c index dece02f257..5ffd6df71e 100644 --- a/src/resmom/pbs_demux.c +++ b/src/resmom/pbs_demux.c @@ -95,13 +95,11 @@ #include #include #include +#include #ifdef ENABLE_BLCR #include #endif /* ENABLE_BLCR */ -#if defined(FD_SET_IN_SYS_SELECT_H) -# include -#endif #include "lib_ifl.h" #include "pbs_helper.h" @@ -114,7 +112,7 @@ struct routem short r_nl; }; -fd_set readset; +struct pollfd *readset = NULL; void readit( @@ -129,6 +127,10 @@ void readit( int i; char *pc; + // confirm readset has been initialized + if (readset == NULL) + return; + if (prm->r_where == old_out) fil = stdout; else @@ -169,7 +171,10 @@ void readit( prm->r_where = invalid; - FD_CLR(sock, &readset); + // remove socket from readset + readset[sock].fd = -1; + readset[sock].events = 0; + readset[sock].revents = 0; } return; @@ -193,7 +198,6 @@ int main( char *argv[]) { - struct timeval timeout; int i; int maxfd; int main_sock_out = 3; @@ -201,7 +205,9 @@ int main( int n; int newsock; pid_t parent; - fd_set selset; + + struct pollfd *pollset; + int pollset_size_bytes; struct routem *routem; @@ -247,7 +253,7 @@ int main( if (routem == NULL) { - perror("cannot alloc memory"); + perror("cannot alloc memory for routem"); exit(5); } @@ -261,9 +267,40 @@ int main( routem[main_sock_out].r_where = new_out; routem[main_sock_err].r_where = new_err; - FD_ZERO(&readset); - FD_SET(main_sock_out, &readset); - FD_SET(main_sock_err, &readset); + pollset_size_bytes = maxfd * sizeof(struct pollfd); + + readset = (struct pollfd *)malloc(pollset_size_bytes); + if (readset == NULL) + { + perror("cannot alloc memory for readset"); + + exit(5); + } + + // initialize readset + for (i = 0; i < maxfd; i++) + { + readset[i].fd = -1; + readset[i].events = 0; + readset[i].revents = 0; + } + + readset[main_sock_out].fd = main_sock_out; + readset[main_sock_out].events = POLLIN; + readset[main_sock_out].revents = 0; + + readset[main_sock_err].fd = main_sock_err; + readset[main_sock_err].events = POLLIN; + readset[main_sock_err].revents = 0; + + // allocate local pollset to hold a copy of readset + pollset = (struct pollfd *)malloc(pollset_size_bytes); + if (pollset == NULL) + { + perror("cannot alloc memory for pollset"); + + exit(5); + } if (listen(main_sock_out, TORQUE_LISTENQUEUE) < 0) { @@ -281,11 +318,11 @@ int main( while (1) { - selset = readset; - timeout.tv_usec = 0; - timeout.tv_sec = 10; + // copy readset to local set for poll + memcpy(pollset, readset, pollset_size_bytes); - n = select(FD_SETSIZE, &selset, (fd_set *)0, (fd_set *)0, &timeout); + // wait for up to 10sec + n = poll(pollset, maxfd, 10000); if (n == -1) { @@ -295,7 +332,7 @@ int main( } else { - fprintf(stderr, "%s: select failed\n", + fprintf(stderr, "%s: poll failed\n", argv[0]); exit(1); @@ -316,12 +353,14 @@ int main( } } /* END else if (n == 0) */ - for (i = 0;(n != 0) && (i < maxfd);++i) + for (i = 0; (n > 0) && (i < maxfd); i++) { - if (FD_ISSET(i, &selset)) + // decrement count of structures with non-zero return events + n--; + + if ((pollset[i].revents & POLLIN)) { /* this socket has data */ - n--; switch ((routem + i)->r_where) { @@ -336,7 +375,10 @@ int main( old_out : old_err; - FD_SET(newsock, &readset); + // add new socket to readset for future polling + readset[newsock].fd = newsock; + readset[newsock].events = POLLIN; + readset[newsock].revents = 0; break; diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index d3daab21cd..582392507a 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -88,6 +88,7 @@ extern "C" #include #include #include +#include #include #include #include @@ -9212,10 +9213,9 @@ int TMomCheckJobChild( { int i; - fd_set fdset; + struct pollfd PollArray; int rc; int read_size = sizeof(struct startjob_rtn); - struct timeval timeout; /* NOTE: assume if anything is on pipe, everything is on pipe (may reasult in hang) */ @@ -9225,24 +9225,17 @@ int TMomCheckJobChild( /* read returns the session id or error */ - timeout.tv_sec = Timeout; - timeout.tv_usec = 0; errno = 0; - FD_ZERO(&fdset); + PollArray.fd = TJE->jsmpipe[0]; + PollArray.events = POLLIN; + PollArray.revents = 0; - FD_SET(TJE->jsmpipe[0], &fdset); + rc = poll(&PollArray, 1, Timeout * 1000); - rc = select( - TJE->jsmpipe[0] + 1, - &fdset, - (fd_set *)NULL, - (fd_set *)NULL, - &timeout); - - if (rc <= 0) + if ((rc <= 0) || ((PollArray.revents & POLLIN) == 0)) { - /* TIMEOUT - data not yet available */ + /* error, timeout or no data not yet available */ return(FAILURE); } diff --git a/src/test/mail_throttler/scaffolding.c b/src/test/mail_throttler/scaffolding.c index c10bce7cee..4de0cb0936 100644 --- a/src/test/mail_throttler/scaffolding.c +++ b/src/test/mail_throttler/scaffolding.c @@ -1,3 +1,4 @@ #include -int pthread_mutex_lock(pthread_mutex_t *mutex) { return(0); } +int pthread_mutex_lock(pthread_mutex_t *mutex) throw() + { return(0); } diff --git a/src/test/mom_comm/test_uut.c b/src/test/mom_comm/test_uut.c index 24322773b1..ce0f65b892 100644 --- a/src/test/mom_comm/test_uut.c +++ b/src/test/mom_comm/test_uut.c @@ -41,6 +41,7 @@ int handle_im_poll_job_response(struct tcp_chan *chan, job &pjob, int nodeidx, h received_node *get_received_node_entry(char *str); bool is_nodeid_on_this_host(job *pjob, tm_node_id nodeid); task *find_task_by_pid(job *pjob, int pid); +int readit(int, int); #ifdef PENABLE_LINUX_CGROUPS int get_req_and_task_index_from_local_rank(job *pjob, int local_rank, unsigned int &req_index, unsigned int &task_index); @@ -578,6 +579,13 @@ START_TEST(get_stat_update_interval_test) } END_TEST +START_TEST(test_readit) + { + // readset uninitialized so expect failure + fail_unless(readit(0, 0) == -2); + } +END_TEST + Suite *mom_comm_suite(void) { Suite *s = suite_create("mom_comm_suite methods"); @@ -636,6 +644,10 @@ Suite *mom_comm_suite(void) #endif suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_readit"); + tcase_add_test(tc_core, test_readit); + suite_add_tcase(s, tc_core); + return(s); } diff --git a/src/test/mom_main/test_uut.c b/src/test/mom_main/test_uut.c index 08ec2061e7..e7a504f065 100644 --- a/src/test/mom_main/test_uut.c +++ b/src/test/mom_main/test_uut.c @@ -27,7 +27,7 @@ unsigned long setcudavisibledevices(const char *); void set_report_mom_cuda_visible_devices(std::stringstream &output, char *curr); void read_mom_hierarchy(); int parse_integer_range(const char *range_str, int &start, int &end); -time_t calculate_select_timeout(); +time_t calculate_poll_timeout(); int process_layout_request(tcp_chan *chan); bool should_resend_obit(job *pjob, int diff); void check_job_in_mom_wait(job *pjob); @@ -342,7 +342,7 @@ END_TEST /** - * time_t calculate_select_timeout(void) + * time_t calculate_poll_timeout(void) * Input: * (G) time_t time_now - periodically updated current time. * (G) time_t wait_time - constant systme-dependent value. @@ -351,7 +351,7 @@ END_TEST * (G) time_t last_poll_time - last poll timestamp. * (G) int CheckPollTime - poll interval. */ -START_TEST(calculate_select_timeout_test) +START_TEST(calculate_poll_timeout_test) { ForceServerUpdate = false; @@ -362,7 +362,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; /* 10 seconds till the next status update */ last_poll_time = 90; CheckPollTime = 30; /* 10 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 9); + fail_unless(calculate_poll_timeout() == 9); /* status update is minimum */ time_now = 110; @@ -371,7 +371,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 19; /* 9 seconds till the next status update */ last_poll_time = 90; CheckPollTime = 30; /* 10 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 9); + fail_unless(calculate_poll_timeout() == 9); /* poll is minimum */ time_now = 110; @@ -380,7 +380,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; /* 10 seconds till the next status update */ last_poll_time = 90; CheckPollTime = 29; /* 9 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 9); + fail_unless(calculate_poll_timeout() == 9); /* LastServerUpdateTime is zero */ time_now = 110; @@ -389,7 +389,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; last_poll_time = 90; CheckPollTime = 30; /* 10 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 1); + fail_unless(calculate_poll_timeout() == 1); /* status update is minimum and was needed to be sent some time ago */ time_now = 110; @@ -398,7 +398,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; /* -1 seconds till the next status update */ last_poll_time = 90; CheckPollTime = 30; /* 10 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 1); + fail_unless(calculate_poll_timeout() == 1); /* poll is minimum and was needed to be sent some time ago */ time_now = 110; @@ -407,7 +407,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; /* 10 seconds till the next status update */ last_poll_time = 79; CheckPollTime = 30; /* -1 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 1); + fail_unless(calculate_poll_timeout() == 1); } END_TEST @@ -462,8 +462,8 @@ Suite *mom_main_suite(void) tcase_add_test(tc_core, test_check_job_in_mom_wait); suite_add_tcase(s, tc_core); - tc_core = tcase_create("calculate_select_timeout_test"); - tcase_add_test(tc_core, calculate_select_timeout_test); + tc_core = tcase_create("calculate_poll_timeout_test"); + tcase_add_test(tc_core, calculate_poll_timeout_test); tcase_add_test(tc_core, test_evaluate_job_in_prerun); suite_add_tcase(s, tc_core); diff --git a/src/test/net_client/scaffolding.c b/src/test/net_client/scaffolding.c index 741f38a6da..f6c7ddf638 100644 --- a/src/test/net_client/scaffolding.c +++ b/src/test/net_client/scaffolding.c @@ -1,5 +1,15 @@ #include "license_pbs.h" /* See here for the software license */ #include +#include void log_err(int errnum, const char *func_id, const char *msg) {} +int global_poll_rc = 0; +short global_poll_revents = 0; + +int poll(struct pollfd *fds, nfds_t nfds, int timeout) + { + fds->revents = global_poll_revents; + + return(global_poll_rc); + } diff --git a/src/test/net_client/test_uut.c b/src/test/net_client/test_uut.c index e4714543b5..39ce006851 100644 --- a/src/test/net_client/test_uut.c +++ b/src/test/net_client/test_uut.c @@ -3,14 +3,32 @@ #include "test_net_client.h" #include #include +#include #include "pbs_error.h" -START_TEST(test_one) +bool wait_for_write_ready(int, int); + +extern int global_poll_rc; +extern short global_poll_revents; + +START_TEST(test_wait_for_write_ready) { + global_poll_rc = 0; + fail_unless(wait_for_write_ready(0, 0) == false); + + global_poll_rc = -1; + fail_unless(wait_for_write_ready(0, 0) == false); + + global_poll_rc = 1; + global_poll_revents = 0; + fail_unless(wait_for_write_ready(0, 0) == false); + global_poll_rc = 1; + global_poll_revents = POLLOUT; + fail_unless(wait_for_write_ready(0, 0) == true); } END_TEST @@ -24,8 +42,8 @@ END_TEST Suite *net_client_suite(void) { Suite *s = suite_create("net_client_suite methods"); - TCase *tc_core = tcase_create("test_one"); - tcase_add_test(tc_core, test_one); + TCase *tc_core = tcase_create("test_wait_for_write_ready"); + tcase_add_test(tc_core, test_wait_for_write_ready); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_two"); diff --git a/src/test/net_common/scaffolding.c b/src/test/net_common/scaffolding.c index a10f1b835f..f17299b9c9 100644 --- a/src/test/net_common/scaffolding.c +++ b/src/test/net_common/scaffolding.c @@ -1,11 +1,19 @@ #include "license_pbs.h" /* See here for the software license */ #include #include +#include +#include +#include bool socket_success = true; bool close_success = true; bool connect_success = true; +int global_poll_rc = 0; +short global_poll_revents = 0; +int global_getsockopt_rc = 0; +int global_sock_errno = 0; + struct addrinfo * insert_addr_name_info(struct addrinfo *pAddrInfo,const char *host) { @@ -76,4 +84,19 @@ int connect( return(-1); } +int poll(struct pollfd *fds, nfds_t nfds, int timeout) + { + fds->revents = global_poll_revents; + + return(global_poll_rc); + } + +int getsockopt(int sockfd, int level, int optname, void *optval, socklen_t *optlen) throw() + { + int *iptr; + + iptr = (int *)optval; + *iptr = global_sock_errno; + return(global_getsockopt_rc); + } diff --git a/src/test/net_common/test_uut.c b/src/test/net_common/test_uut.c index c97749fa1f..e5209d7087 100644 --- a/src/test/net_common/test_uut.c +++ b/src/test/net_common/test_uut.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "pbs_error.h" #include "net_cache.h" @@ -15,6 +16,12 @@ bool connect_success; int get_random_reserved_port(); int process_and_save_socket_error(int socket_errno); +int socket_wait_for_write(int); + +extern int global_poll_rc; +extern short global_poll_revents; +extern int global_getsockopt_rc; +extern int global_sock_errno; START_TEST(test_get_random_reserved_port) { @@ -105,6 +112,38 @@ START_TEST(test_socket_connect_unix) } END_TEST +START_TEST(test_socket_wait_for_write) + { + global_poll_rc = 0; + fail_unless(socket_wait_for_write(0) == PERMANENT_SOCKET_FAIL); + + global_poll_rc = -1; + fail_unless(socket_wait_for_write(0) == PERMANENT_SOCKET_FAIL); + + global_poll_rc = 1; + global_poll_revents = 0; + fail_unless(socket_wait_for_write(0) == PERMANENT_SOCKET_FAIL); + + global_poll_rc = 1; + global_poll_revents = POLLOUT; + global_getsockopt_rc = 1; + global_sock_errno = EALREADY; + fail_unless(socket_wait_for_write(0) == TRANSIENT_SOCKET_FAIL); + + global_poll_rc = 1; + global_poll_revents = POLLOUT; + global_getsockopt_rc = 0; + global_sock_errno = EINPROGRESS; + fail_unless(socket_wait_for_write(0) == TRANSIENT_SOCKET_FAIL); + + global_poll_rc = 1; + global_poll_revents = POLLOUT; + global_getsockopt_rc = 0; + global_sock_errno = 0; + fail_unless(socket_wait_for_write(0) == PBSE_NONE); + } +END_TEST + Suite *net_common_suite(void) { Suite *s = suite_create("net_common_suite methods"); @@ -128,6 +167,10 @@ Suite *net_common_suite(void) tcase_add_test(tc_core, test_socket_connect_unix); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_socket_wait_for_write"); + tcase_add_test(tc_core, test_socket_wait_for_write); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/net_server/scaffolding.c b/src/test/net_server/scaffolding.c index b4e1c18b71..cbc4c6d371 100644 --- a/src/test/net_server/scaffolding.c +++ b/src/test/net_server/scaffolding.c @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include "pbs_error.h" #include "net_connect.h" @@ -14,6 +16,9 @@ bool socket_write_success = true; bool socket_read_success = true; bool socket_read_code = true; +nfds_t global_poll_nfds = 0; +int global_poll_timeout_ms = 0; + const char *msg_daemonname = "unset"; void disiui_() {} @@ -109,7 +114,6 @@ void log_event(int eventtype, int objclass, const char *objname, const char *tex void initialize_connections_table() { - fprintf(stderr, "The call to initialize_connections_table needs to be mocked!!\n"); return; } @@ -119,25 +123,13 @@ char *PAddrToString(pbs_net_t *Addr) return(0); } -int get_max_num_descriptors(void) - { - fprintf(stderr, "The call to get_max_num_descriptors needs to be mocked!!\n"); - return(0); - } - -int get_fdset_size(void) - { - fprintf(stderr, "The call to get_fdset_size needs to be mocked!!\n"); - return(0); - } - void log_err(int errnum, const char *routine, const char *text) {} void log_record(int eventtype, int objclass, const char *objname, const char *text) {} int pbs_getaddrinfo(const char *pNode,struct addrinfo *pHints,struct addrinfo **ppAddrInfoOut) { - return(0); + return(1); } char *get_cached_nameinfo(const struct sockaddr_in *sai) @@ -149,3 +141,20 @@ bool log_available(int eventtype) { return true; } + +char *netaddr_long(long ap, char *out) + { + return(NULL); + } + +int get_max_num_descriptors(void) + { + return(getdtablesize()); + } + +int poll(struct pollfd *fds, nfds_t nfds, int timeout) + { + global_poll_nfds = nfds; + global_poll_timeout_ms = timeout; + return(0); + } diff --git a/src/test/net_server/test_uut.c b/src/test/net_server/test_uut.c index 16414af743..4fc8398f89 100644 --- a/src/test/net_server/test_uut.c +++ b/src/test/net_server/test_uut.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "server_limits.h" @@ -21,9 +22,16 @@ extern bool socket_write_success; extern bool socket_read_success; extern bool socket_read_code; +extern nfds_t global_poll_nfds; +extern int global_poll_timeout_ms; + int add_connection(int sock, enum conn_type type, pbs_net_t addr, unsigned int port, unsigned int socktype, void *(*func)(void *), int add_wait_request); void *accept_conn(void *new_conn); +int init_network(unsigned int port, void *(*readfunc)(void *)); +void globalset_add_sock(int, u_long, u_long); +void globalset_del_sock(int); + START_TEST(netaddr_pbs_net_t_test_one) { @@ -145,6 +153,22 @@ START_TEST(test_check_trqauthd_unix_domain_port) } END_TEST +START_TEST(test_init_network) + { + int rc; + int timeout_sec = 10; + nfds_t MaxNumDescriptors = get_max_num_descriptors(); + + rc = init_network(0, NULL); + fail_unless(rc == PBSE_NONE); + + rc = wait_request(timeout_sec, NULL); + + // examine the values passed to poll() to see if they were expected + fail_unless(global_poll_nfds == MaxNumDescriptors); + fail_unless(global_poll_timeout_ms == timeout_sec * 1000); + } +END_TEST Suite *net_server_suite(void) { @@ -169,6 +193,10 @@ Suite *net_server_suite(void) tcase_add_test(tc_core, test_check_trqauthd_unix_domain_port); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_init_network"); + tcase_add_test(tc_core, test_init_network); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/pbs_demux/test_uut.c b/src/test/pbs_demux/test_uut.c index a7f8fa44b1..380d052ebd 100644 --- a/src/test/pbs_demux/test_uut.c +++ b/src/test/pbs_demux/test_uut.c @@ -6,10 +6,14 @@ #include "pbs_error.h" -START_TEST(test_one) - { - +void readit(int, struct routem*); +START_TEST(test_readit) + { + // this is a void function but expect + // immediate return with no segfault + // since readset not initialized + readit(0, NULL); } END_TEST @@ -23,8 +27,8 @@ END_TEST Suite *pbs_demux_suite(void) { Suite *s = suite_create("pbs_demux_suite methods"); - TCase *tc_core = tcase_create("test_one"); - tcase_add_test(tc_core, test_one); + TCase *tc_core = tcase_create("test_readit"); + tcase_add_test(tc_core, test_readit); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_two"); diff --git a/src/test/qsub_functions/scaffolding.c b/src/test/qsub_functions/scaffolding.c index 4e21c50f7f..9e1d24df43 100644 --- a/src/test/qsub_functions/scaffolding.c +++ b/src/test/qsub_functions/scaffolding.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "u_hash_map_structs.h" #include "port_forwarding.h" #include "req.hpp" @@ -30,6 +31,10 @@ bool find_size = false; bool validate_path = false; int req_val = 0; +int global_poll_rc = 0; +short global_poll_revents = 0; +int global_poll_errno = 0; + std::string added_value; std::string added_name; std::vector in_hash; @@ -480,4 +485,10 @@ struct group *getgrnam_ext( return(grp); } /* END getgrnam_ext() */ +int poll(struct pollfd *fds, nfds_t nfds, int timeout) + { + fds->revents = global_poll_revents; + errno = global_poll_errno; + return(global_poll_rc); + } diff --git a/src/test/qsub_functions/test_uut.c b/src/test/qsub_functions/test_uut.c index ab5a60c539..93ba24c230 100644 --- a/src/test/qsub_functions/test_uut.c +++ b/src/test/qsub_functions/test_uut.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "qsub_functions.h" #include "test_qsub_functions.h" @@ -20,6 +21,7 @@ int process_opt_k(job_info *ji, const char *cmd_arg, int data_type); int process_opt_K(job_info *ji, const char *cmd_arg, int data_type); int process_opt_m(job_info *ji, const char *cmd_arg, int data_type); int process_opt_p(job_info *ji, const char *cmd_arg, int data_type); +int wait_for_read_ready(int, int); extern complete_req cr; extern bool submission_string_fail; @@ -31,6 +33,10 @@ extern bool validate_path; extern std::string added_value; extern std::string added_name; +extern int global_poll_rc; +extern short global_poll_revents; +extern int global_poll_errno; + bool are_we_forking() { @@ -378,6 +384,44 @@ START_TEST(test_make_argv) } END_TEST +START_TEST(test_wait_for_read_ready) + { + int rc; + int some_fd = 0; // arbitrary value + int some_timeout_sec = 1; // arbitrary value > 0 + + // emulate timeout + global_poll_rc = 0; + global_poll_revents = 0; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == 0); + + // emulate ready to read + global_poll_rc = 1; + global_poll_revents = POLLIN; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == 1); + + // emulate not ready to read + global_poll_rc = 1; + global_poll_revents = 0; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == 0); + + // emulate failure + global_poll_rc = -1; + global_poll_errno = EFAULT; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == -1); + + // emulate recoverable failure + global_poll_rc = -1; + global_poll_errno = EINTR; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == 0); + } + +END_TEST Suite *qsub_functions_suite(void) { Suite *s = suite_create("qsub_functions methods"); @@ -412,6 +456,10 @@ Suite *qsub_functions_suite(void) tcase_add_test(tc_core, test_is_resource_request_valid); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_wait_for_read_ready"); + tcase_add_test(tc_core, test_wait_for_read_ready); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/rm/scaffolding.c b/src/test/rm/scaffolding.c index 6fc4ff0970..8a40b03112 100644 --- a/src/test/rm/scaffolding.c +++ b/src/test/rm/scaffolding.c @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include "dis.h" #include "tcp.h" @@ -87,14 +89,7 @@ int connect(int sock,const sockaddr *addr,socklen_t addrLen) int get_max_num_descriptors() { - fprintf(stderr, "The call to get_max_num_descriptors needs to be mocked!!\n"); - exit(1); - } - -int get_fdset_size() - { - fprintf(stderr, "The call to get_fdset_size needs to be mocked!!\n"); - exit(1); + return(getdtablesize()); } char *pbs_strerror(int err) @@ -110,3 +105,8 @@ int pbs_getaddrinfo(const char *pNode,struct addrinfo *pHints,struct addrinfo ** (*ppAddrInfoOut)->ai_addr = (struct sockaddr *)calloc(1,sizeof(struct sockaddr_in)); return(0); } + +int poll(struct pollfd *fds, nfds_t nfds, int timeout) + { + return(0); + } diff --git a/src/test/rm/test_uut.c b/src/test/rm/test_uut.c index f05770c000..f8e6ddfb93 100644 --- a/src/test/rm/test_uut.c +++ b/src/test/rm/test_uut.c @@ -11,7 +11,6 @@ extern int debug_read(int sock,char **bf,long long *len); extern int debug_write(int sock,char *bf,long long len); - START_TEST(test_addreq) { @@ -123,6 +122,15 @@ START_TEST(test_getreq) } END_TEST +START_TEST(test_activereq) + { + int rc; + + rc = activereq(); + fail_unless(rc == -2); + } +END_TEST + Suite *rm_suite(void) { Suite *s = suite_create("rm_suite methods"); @@ -150,6 +158,10 @@ Suite *rm_suite(void) tcase_add_test(tc_core, test_getreq); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_activereq"); + tcase_add_test(tc_core, test_activereq); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/start_exec/scaffolding.c b/src/test/start_exec/scaffolding.c index 982ac0c5e2..ce2a2a5f48 100644 --- a/src/test/start_exec/scaffolding.c +++ b/src/test/start_exec/scaffolding.c @@ -11,6 +11,7 @@ #include /* std::string */ #include #include +#include #include "attribute.h" /* attribute_def, pbs_attribute, svrattrl */ #include "resource.h" /* resource_def */ @@ -104,6 +105,9 @@ int job_saved; int task_saved; std::string presetup_prologue; +int global_poll_fd = -1; +int global_poll_timeout_ms = -1; + #ifdef NUMA_SUPPORT nodeboard node_boards[MAX_NODE_BOARDS]; int num_node_boards = 10; @@ -1101,3 +1105,14 @@ void register_jobs_nspace(job *pjob, pjobexec_t *TJE) {} int setup_gpus_for_job(job *pjob) {return(0);} + +int poll(struct pollfd *fds, nfds_t nfds, int timeout) + { + if (nfds > 0) + { + global_poll_fd = fds[0].fd; + global_poll_timeout_ms = timeout; + } + + return(0); + } diff --git a/src/test/start_exec/test_uut.c b/src/test/start_exec/test_uut.c index 56409588ee..3e45eb7b16 100644 --- a/src/test/start_exec/test_uut.c +++ b/src/test/start_exec/test_uut.c @@ -45,6 +45,9 @@ extern bool fail_site_grp_check; extern bool am_ms; extern bool addr_fail; +extern int global_poll_fd; +extern int global_poll_timeout_ms; + void create_command(std::string &cmd, char **argv); void no_hang(int sig); void exec_bail(job *pjob, int code, std::set *sisters_contacted); @@ -53,6 +56,7 @@ int process_launcher_child_status(struct startjob_rtn *sjr, const char *job_id, void update_task_and_job_states_after_launch(task *ptask, job *pjob, const char *application_name); void escape_spaces(const char *str, std::string &escaped); int become_the_user(job*, bool); +int TMomCheckJobChild(pjobexec_t*, int, int*, int*); #if NO_SPOOL_OUTPUT == 1 int save_supplementary_group_list(int*, gid_t**); @@ -763,6 +767,21 @@ START_TEST(test_become_the_user) } END_TEST +START_TEST(test_TMomCheckJobChild) + { + pjobexec_t TJE; + int timeout_sec = 39; + int count; + int rc; + int fd = 99; + + TJE.jsmpipe[0] = fd; + TMomCheckJobChild(&TJE, timeout_sec, &count, &rc); + fail_unless(global_poll_fd == fd); + fail_unless(global_poll_timeout_ms == timeout_sec * 1000); + } +END_TEST + Suite *start_exec_suite(void) { Suite *s = suite_create("start_exec_suite methods"); @@ -828,6 +847,10 @@ Suite *start_exec_suite(void) tcase_add_test(tc_core, test_become_the_user); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_TMomCheckJobChild"); + tcase_add_test(tc_core, test_TMomCheckJobChild); + suite_add_tcase(s, tc_core); + return s; } From 3a9b4439a9389627ec81b15b7ba9bb1d7bc3f448 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 7 Dec 2016 11:40:06 -0700 Subject: [PATCH 092/312] TRQ-3862. Fix a regression caused by this change. --- src/cmds/qsub_functions.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index a6f2fbe4c5..5a65d34659 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -3397,7 +3397,8 @@ void process_opts( if (dependency_options.size() > 1) { alternate_dependency = strdup(dependency_options[1].c_str()); - alternate_data_type = data_type; + // Make this overwrite the previous value if needed + alternate_data_type = data_type - 1; } } else if (!strcmp(keyword, ATTR_job_radix)) From 42769b6aa708a7e7453be3790e2455955cf270f4 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 7 Dec 2016 15:50:17 -0700 Subject: [PATCH 093/312] TRQ-3822. Make the default gpu mode apply to -L jobs as well. --- src/include/req.hpp | 2 +- src/lib/Libattr/req.cpp | 8 +----- src/lib/Libutils/allocation.cpp | 4 +-- src/lib/Libutils/machine.cpp | 2 +- src/lib/Libutils/numa_chip.cpp | 4 +-- src/resmom/nvidia.c | 2 +- src/server/node_manager.c | 47 ++++++++++++++++++++++++++++++- src/test/allocation/scaffolding.c | 2 +- src/test/machine/scaffolding.c | 2 +- src/test/numa_chip/scaffolding.c | 2 +- src/test/req/test_uut.c | 6 ++-- 11 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/include/req.hpp b/src/include/req.hpp index 7220238926..441c64da7c 100644 --- a/src/include/req.hpp +++ b/src/include/req.hpp @@ -182,7 +182,7 @@ class req int getPlaceThreads() const; unsigned long getDisk() const; int getMaxtpn() const; - int getGpus() const; + int get_gpus() const; int getMics() const; std::string getGpuMode() const; std::string getGres() const; diff --git a/src/lib/Libattr/req.cpp b/src/lib/Libattr/req.cpp index ede978d970..c8d78262d0 100644 --- a/src/lib/Libattr/req.cpp +++ b/src/lib/Libattr/req.cpp @@ -2294,12 +2294,6 @@ int req::getMaxtpn() const return(this->maxtpn); } -std::string req::getGpuMode() const - - { - return(this->gpu_mode); - } - std::string req::getReqAttr() const { @@ -2318,7 +2312,7 @@ int req::getIndex() const return(this->index); } -int req::getGpus() const +int req::get_gpus() const { return(this->gpus); diff --git a/src/lib/Libutils/allocation.cpp b/src/lib/Libutils/allocation.cpp index 1349b84a0c..eca35b0583 100644 --- a/src/lib/Libutils/allocation.cpp +++ b/src/lib/Libutils/allocation.cpp @@ -158,7 +158,7 @@ allocation::allocation( { this->cpus = r.getExecutionSlots(); this->memory = r.getMemory(); - this->gpus = r.getGpus(); + this->gpus = r.get_gpus(); this->mics = r.getMics(); if (r.getThreadUsageString() == use_cores) @@ -555,7 +555,7 @@ bool allocation::partially_placed( { return ((this->cpus != r.getExecutionSlots()) || (this->memory != r.getMemory()) || - (this->gpus != r.getGpus()) || + (this->gpus != r.get_gpus()) || (this->mics != r.getMics()) || ((r.getPlaceCores() > 0) && (this->place_cpus != r.getPlaceCores())) || diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index 9be3c29e00..94ba768906 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -780,7 +780,7 @@ int Machine::spread_place_pu( pu_per_task_remaining = pu_per_task; lprocs_per_task_remaining = lprocs_per_task; - gpus_remaining = r.getGpus(); + gpus_remaining = r.get_gpus(); mics_remaining = r.getMics(); for (unsigned int j = 0; j < this->totalSockets; j++) diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index 1b8f8a96a2..79d285f697 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -978,7 +978,7 @@ double Chip::how_many_tasks_fit( else mem_tasks = cpu_tasks; - double gpus = r.getGpus(); + double gpus = r.get_gpus(); if (gpus > 0) { gpu_tasks = this->available_gpus / gpus; @@ -1370,7 +1370,7 @@ bool Chip::task_will_fit( bool fits = false; int max_cpus = r.getExecutionSlots(); hwloc_uint64_t mem_per_task = r.getMemory(); - int gpus_per_task = r.getGpus(); + int gpus_per_task = r.get_gpus(); int mics_per_task = r.getMics(); bool cores_only = (r.getThreadUsageString() == use_cores); diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index b0eb4d4779..628acc9755 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -1122,7 +1122,7 @@ int set_gpu_req_modes( /* Look through each req and see if there is a mode to set */ r = cr->get_req(i); gpu_mode = r.get_gpu_mode(); /* returns a string indicating the gpu mode */ - total_req_gpus = r.getGpus(); + total_req_gpus = r.get_gpus(); for (unsigned int j = 0; j < total_req_gpus && it != gpu_indices.end(); j++) { diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 0884bc626a..6fdf2a4cf3 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -4157,6 +4157,49 @@ void update_req_hostlist( } } // END update_req_hostlist() + + + +/* + * set_gpu_mode_if_needed() + * Sets the gpu mode if there's a default gpu mode, this job requests gpus, and this job + * doesn't set the gpu mode + * + * @param pjob - the job whose gpu mode we may set + */ + +void set_gpu_mode_if_needed( + + job *pjob) + + { + char *default_gpu_mode = NULL; + + if (pjob->ji_wattr[JOB_ATR_request_version].at_val.at_long > 1) + { + complete_req *cr = (complete_req *)pjob->ji_wattr[JOB_ATR_req_information].at_val.at_ptr; + + if (cr != NULL) + { + get_svr_attr_str(SRV_ATR_DefaultGpuMode, &default_gpu_mode); + + if (default_gpu_mode != NULL) + { + for (unsigned int i = 0; i < cr->get_num_reqs(); i++) + { + req &r = cr->get_req(i); + + if ((r.get_gpus() > 0) && + (r.get_gpu_mode().size() == 0)) + { + r.set_attribute(default_gpu_mode); + } + } + } + } + } + + } // END set_gpu_mode_if_needed() #endif @@ -4220,6 +4263,8 @@ int place_subnodes_in_hostlist( bool legacy_vmem = false; get_svr_attr_b(SRV_ATR_LegacyVmem, &legacy_vmem); + set_gpu_mode_if_needed(pjob); + // We shouldn't be starting a job if the layout hasn't been set up yet. if (pnode->nd_layout.is_initialized() == false) return(-1); @@ -4726,7 +4771,7 @@ void add_entry_to_naji_list( naji.node_id = pnode->nd_id; naji.ppn_needed = r.get_execution_slots() * tasks_placed; - naji.gpu_needed = r.getGpus() * tasks_placed; + naji.gpu_needed = r.get_gpus() * tasks_placed; naji.mic_needed = r.getMics() * tasks_placed; naji.is_external = false; diff --git a/src/test/allocation/scaffolding.c b/src/test/allocation/scaffolding.c index a6217771a5..acca1c2788 100644 --- a/src/test/allocation/scaffolding.c +++ b/src/test/allocation/scaffolding.c @@ -45,7 +45,7 @@ int req::getMics() const return(0); } -int req::getGpus() const +int req::get_gpus() const { return(0); diff --git a/src/test/machine/scaffolding.c b/src/test/machine/scaffolding.c index 32a792e8a5..5774cbd41b 100644 --- a/src/test/machine/scaffolding.c +++ b/src/test/machine/scaffolding.c @@ -385,7 +385,7 @@ int req::getMics() const return(0); } -int req::getGpus() const +int req::get_gpus() const { return(0); } diff --git a/src/test/numa_chip/scaffolding.c b/src/test/numa_chip/scaffolding.c index 09a71bf45d..51089f7fb7 100644 --- a/src/test/numa_chip/scaffolding.c +++ b/src/test/numa_chip/scaffolding.c @@ -189,7 +189,7 @@ int req::getMics() const return(this->mics); } -int req::getGpus() const +int req::get_gpus() const { return(this->gpus); } diff --git a/src/test/req/test_uut.c b/src/test/req/test_uut.c index bd910ba5d5..446e0c02b5 100644 --- a/src/test/req/test_uut.c +++ b/src/test/req/test_uut.c @@ -69,7 +69,7 @@ START_TEST(test_get_gpu_mode) r.set_value("gpus", "1", false); r.set_value("gpu_mode", "exclusive_thread", false); - fail_unless(r.getGpus() == 1); + fail_unless(r.get_gpus() == 1); gpu_mode = r.get_gpu_mode(); fail_unless(gpu_mode.compare("exclusive_thread") == 0); } @@ -136,7 +136,7 @@ START_TEST(test_string_constructor) req r3(strdup("2:ppn=8:fast:gpus=2")); fail_unless(r3.getTaskCount() == 2); fail_unless(r3.getExecutionSlots() == 8); - fail_unless(r3.getGpus() == 2); + fail_unless(r3.get_gpus() == 2); } END_TEST @@ -465,7 +465,7 @@ START_TEST(test_equals_operator) fail_unless(r2.getExecutionSlots() == ALL_EXECUTION_SLOTS); fail_unless(r2.getMemory() == 1024 * 1024 * 1024); fail_unless(r2.getMaxtpn() == 4); - fail_unless(r2.getGpuMode() == "exclusive_thread"); + fail_unless(r2.get_gpu_mode() == "exclusive_thread"); fail_unless(r2.getReqAttr() == "matlab>=7", "reqattr: '%s'", r2.getReqAttr().c_str()); // make sure this doesn't segfault From b8353e8b4cde64ab6286ecf3ca89f746abae75f6 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 9 Dec 2016 10:59:03 -0700 Subject: [PATCH 094/312] TRQ-3685. Make jobs with array dependencies that can never be satisfied get deleted. Just like non-array dependencies. Conflicts: src/server/array_func.c src/server/req_delete.c src/server/req_register.c src/test/req_delete/scaffolding.c src/test/req_delete/test_uut.c --- src/server/array_func.c | 4 + src/server/req_delete.c | 119 ++++++++++++++-------------- src/server/req_register.c | 93 ++++++++++++---------- src/test/req_delete/scaffolding.c | 85 +++----------------- src/test/req_delete/test_uut.c | 21 +++++ src/test/req_register/scaffolding.c | 3 +- src/test/req_register/test_uut.c | 46 ++++++++++- 7 files changed, 192 insertions(+), 179 deletions(-) diff --git a/src/server/array_func.c b/src/server/array_func.c index a1cf0d88b6..b27dcf03e0 100644 --- a/src/server/array_func.c +++ b/src/server/array_func.c @@ -1482,6 +1482,7 @@ int delete_array_range( } } + pa->ai_qs.jobs_done += num_deleted; pa->ai_qs.num_failed += num_deleted; return(num_skipped); @@ -1618,7 +1619,10 @@ int delete_whole_array( } if (pa != NULL) + { pa->ai_qs.num_failed += num_deleted; + pa->ai_qs.jobs_done += num_deleted; + } if (num_jobs == 0) return(NO_JOBS_IN_ARRAY); diff --git a/src/server/req_delete.c b/src/server/req_delete.c index 4285e4a8a1..d9cb3ab012 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -319,10 +319,57 @@ void remove_stagein( +/* + * perform_job_delete_array_bookkeeping() + * + * Updates the array values to account for pjob's deletion if pjob is an array subjob + * @param pjob - the job being deleted + * @param cancel_exit_code - the exit code for this job + * @return PBSE_NONE on success, PBSE_JOBNOTFOUND if the job disappears + */ + +int perform_job_delete_array_bookkeeping( + + job *pjob, + int cancel_exit_code) + + { + int rc = PBSE_NONE; + + if ((pjob->ji_arraystructid[0] != '\0') && + (pjob->ji_is_array_template == FALSE)) + { + job_array *pa = get_jobs_array(&pjob); + + if (pjob == NULL) + return(PBSE_JOBNOTFOUND); + + int old_state = pjob->ji_qs.ji_state; + std::string jobid(pjob->ji_qs.ji_jobid); + + unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); + + pa->update_array_values(old_state, + aeTerminate, + jobid.c_str(), + cancel_exit_code); + + if ((pjob = svr_find_job((char *)jobid.c_str(),FALSE)) == NULL) + rc = PBSE_JOBNOTFOUND; + + unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); + } + + return(rc); + } // END perform_job_delete_array_bookkeeping() + + + /* * force_purge_work() * - * Performs the work of purging an individual job (qdel -p) + * Performs the work of purging an individual job (qdel -p). The job comes in locked, + * but is never locked when this function exits. * @param pjob - the job that will be purged */ @@ -352,31 +399,14 @@ void force_purge_work( } } - depend_on_term(pjob); - - if ((pjob->ji_arraystructid[0] != '\0') && - (pjob->ji_is_array_template == false)) + // On error, the job has disappeared, which should happen once we're done with this function + // anyway. + if (perform_job_delete_array_bookkeeping(pjob, cancel_exit_code) == PBSE_NONE) { - job_array *pa = get_jobs_array(&pjob); - - if (pjob == NULL) - throw (int)PBSE_JOB_RECYCLED; - - if (pa != NULL) - { - pa->update_array_values(pjob->ji_qs.ji_state, - aeTerminate, - pjob->ji_qs.ji_jobid, - cancel_exit_code); - - unlock_ai_mutex(pa, __func__, "", LOGLEVEL); - } - } - - svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE); + depend_on_term(pjob); + + svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE); - if (pjob != NULL) - { if (is_ms_on_server(pjob)) { char log_buf[LOCAL_LOG_BUF_SIZE]; @@ -390,6 +420,9 @@ void force_purge_work( else svr_job_purge(pjob); } + else + throw (int)PBSE_JOB_RECYCLED; + } /* END force_purge_work() */ @@ -689,43 +722,9 @@ int execute_job_delete( pjob->ji_wattr[JOB_ATR_exitstat].at_flags |= ATR_VFLAG_SET; } - // Update the array book-keeping values if this is an array subjob - if ((pjob->ji_arraystructid[0] != '\0') && - (pjob->ji_is_array_template == false)) - { - job_array *pa = get_jobs_array(&pjob); - - if (pjob == NULL) - { - job_mutex.set_unlock_on_exit(false); - return(-1); - } - - std::string dup_job_id(pjob->ji_qs.ji_jobid); - - if (pa != NULL) - { - if (pjob != NULL) - { - if (pjob->ji_qs.ji_state != JOB_STATE_RUNNING) - { - int job_exit_status = pjob->ji_qs.ji_un.ji_exect.ji_exitstat; - int job_state = pjob->ji_qs.ji_state; - - job_mutex.unlock(); - pa->update_array_values(job_state, aeTerminate, dup_job_id.c_str(), job_exit_status); - - if ((pjob = svr_find_job((char *)dup_job_id.c_str(),FALSE)) != NULL) - job_mutex.mark_as_locked(); - } - } - - unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); - } - } - - if (pjob == NULL) + if (perform_job_delete_array_bookkeeping(pjob, status_cancel_queue) != PBSE_NONE) { + // The job disappeared while updating the array job_mutex.set_unlock_on_exit(false); return -1; } diff --git a/src/server/req_register.c b/src/server/req_register.c index 26754c02aa..cd2a9411f2 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -1130,11 +1130,13 @@ bool set_array_depend_holds( job_array *pa) { - int compareNumber; + int compare_number; + int could_fulfill_dependency; bool dependency_satisfied = false; job *pjob; array_depend_job *pdj; + char log_buf[LOCAL_LOG_BUF_SIZE]; /* loop through dependencies to update holds */ for (std::list::iterator it = pa->ai_qs.deps.begin(); @@ -1142,55 +1144,38 @@ bool set_array_depend_holds( { array_depend *pdep = *it; - compareNumber = -1; + compare_number = -1; switch (pdep->dp_type) { case JOB_DEPEND_TYPE_AFTERSTARTARRAY: - - compareNumber = pa->ai_qs.num_started; - - break; - - case JOB_DEPEND_TYPE_AFTEROKARRAY: - - compareNumber = pa->ai_qs.num_successful; - - break; - - case JOB_DEPEND_TYPE_AFTERNOTOKARRAY: - - compareNumber = pa->ai_qs.num_failed; - - break; - - case JOB_DEPEND_TYPE_AFTERANYARRAY: - - compareNumber = pa->ai_qs.jobs_done; - - break; - case JOB_DEPEND_TYPE_BEFORESTARTARRAY: - compareNumber = pa->ai_qs.num_started; + compare_number = pa->ai_qs.num_started; + could_fulfill_dependency = pa->ai_qs.num_jobs - pa->ai_qs.jobs_done; break; + case JOB_DEPEND_TYPE_AFTEROKARRAY: case JOB_DEPEND_TYPE_BEFOREOKARRAY: - compareNumber = pa->ai_qs.num_successful; + compare_number = pa->ai_qs.num_successful; + could_fulfill_dependency = pa->ai_qs.num_jobs - pa->ai_qs.num_failed; break; + case JOB_DEPEND_TYPE_AFTERNOTOKARRAY: case JOB_DEPEND_TYPE_BEFORENOTOKARRAY: - compareNumber = pa->ai_qs.num_failed; + compare_number = pa->ai_qs.num_failed; + could_fulfill_dependency = pa->ai_qs.num_jobs - pa->ai_qs.num_successful; break; + case JOB_DEPEND_TYPE_AFTERANYARRAY: case JOB_DEPEND_TYPE_BEFOREANYARRAY: - compareNumber = pa->ai_qs.jobs_done; + compare_number = pa->ai_qs.jobs_done; break; } @@ -1207,25 +1192,49 @@ bool set_array_depend_holds( { mutex_mgr job_mutex(pjob->ji_mutex, true); - if (((compareNumber < pdj->dc_num) && + if (((compare_number < pdj->dc_num) && (pdep->dp_type < JOB_DEPEND_TYPE_BEFORESTARTARRAY)) || - ((compareNumber >= pdj->dc_num) && + ((compare_number >= pdj->dc_num) && (pdep->dp_type > JOB_DEPEND_TYPE_AFTERANYARRAY))) { - /* hold */ - pjob->ji_wattr[JOB_ATR_hold].at_val.at_long |= HOLD_s; - pjob->ji_wattr[JOB_ATR_hold].at_flags |= ATR_VFLAG_SET; - - if (LOGLEVEL >= 8) + // AFTERANYARRAY can always be potentially fulfilled, and any BEFORE* dependency + // that isn't satisfied means it can never be fulfilled + if ((pdep->dp_type > JOB_DEPEND_TYPE_AFTERANYARRAY) || + ((pdep->dp_type < JOB_DEPEND_TYPE_AFTERANYARRAY) && + (could_fulfill_dependency < pdj->dc_num))) { - log_event( - PBSEVENT_JOB, - PBS_EVENTCLASS_JOB, - pjob->ji_qs.ji_jobid, - "Setting HOLD_s due to dependencies\n"); + // These are dependencies that can never be fulfilled + if ((pjob->ji_qs.ji_state < JOB_STATE_EXITING) && + (pjob->ji_qs.ji_state != JOB_STATE_RUNNING)) + { + sprintf(log_buf, + "Job %s deleted because its dependency of array %s can never be satisfied", + pjob->ji_qs.ji_jobid, pa->ai_qs.parent_id); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf); + + /* pjob freed and set to NULL */ + job_abt(&pjob, log_buf, true); + if (pjob == NULL) + job_mutex.set_unlock_on_exit(false); + } } + else + { + // hold + pjob->ji_wattr[JOB_ATR_hold].at_val.at_long |= HOLD_s; + pjob->ji_wattr[JOB_ATR_hold].at_flags |= ATR_VFLAG_SET; - svr_setjobstate(pjob, JOB_STATE_HELD, JOB_SUBSTATE_DEPNHOLD, FALSE); + if (LOGLEVEL >= 8) + { + log_event( + PBSEVENT_JOB, + PBS_EVENTCLASS_JOB, + pjob->ji_qs.ji_jobid, + "Setting HOLD_s due to dependencies\n"); + } + + svr_setjobstate(pjob, JOB_STATE_HELD, JOB_SUBSTATE_DEPNHOLD, FALSE); + } } else { diff --git a/src/test/req_delete/scaffolding.c b/src/test/req_delete/scaffolding.c index b5cf59b128..298ecb02cf 100644 --- a/src/test/req_delete/scaffolding.c +++ b/src/test/req_delete/scaffolding.c @@ -37,6 +37,8 @@ int nanny = 1; bool br_freed; int alloc_work = 1; int depend_term_called; +int updated_array_values = 0; +bool find_job_fail = false; batch_request *alloc_br(int type) { @@ -48,7 +50,10 @@ batch_request *alloc_br(int type) job_array *get_jobs_array(job **pjob) { - return(NULL); + if (pjob == NULL) + return(NULL); + + return((job_array *)calloc(1, sizeof(job_array))); } void account_record(int acctype, job *pjob, const char *text) @@ -168,7 +173,9 @@ int svr_setjobstate(job *pjob, int newstate, int newsubstate, int has_queue_mut job *svr_find_job(const char *jobid, int get_subjob) { - if (strcmp(jobid, "1.napali") == 0) + if ((strcmp(jobid, "1.napali") == 0) || + ((strstr(jobid, "roshar") != NULL) && + (find_job_fail == false))) { job *pjob = new job(); strcpy(pjob->ji_qs.ji_jobid, jobid); @@ -352,40 +359,6 @@ int lock_ji_mutex( { int rc = PBSE_NONE; - char *err_msg = NULL; - char stub_msg[] = "no pos"; - - if (logging >= 10) - { - err_msg = (char *)calloc(1, MSG_LEN_LONG); - if (msg == NULL) - msg = stub_msg; - snprintf(err_msg, MSG_LEN_LONG, "locking %s in method %s-%s", pjob->ji_qs.ji_jobid, id, msg); - log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); - } - - if (pjob->ji_mutex != NULL) - { - if (pthread_mutex_lock(pjob->ji_mutex) != 0) - { - if (logging >= 20) - { - snprintf(err_msg, MSG_LEN_LONG, "ALERT: cannot lock job %s mutex in method %s", - pjob->ji_qs.ji_jobid, id); - log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); - } - rc = PBSE_MUTEX; - } - } - else - { - rc = -1; - log_err(rc, __func__, "Uninitialized mutex pass to pthread_mutex_lock!"); - } - - if (err_msg != NULL) - free(err_msg); - return rc; } @@ -399,42 +372,8 @@ int unlock_ji_mutex( { int rc = PBSE_NONE; - char *err_msg = NULL; - char stub_msg[] = "no pos"; - - if (logging >= 10) - { - err_msg = (char *)calloc(1, MSG_LEN_LONG); - if (msg == NULL) - msg = stub_msg; - snprintf(err_msg, MSG_LEN_LONG, "unlocking %s in method %s-%s", pjob->ji_qs.ji_jobid, id, msg); - log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); - } - - if (pjob->ji_mutex != NULL) - { - if (pthread_mutex_unlock(pjob->ji_mutex) != 0) - { - if (logging >= 20) - { - snprintf(err_msg, MSG_LEN_LONG, "ALERT: cannot unlock job %s mutex in method %s", - pjob->ji_qs.ji_jobid, id); - log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); - } - rc = PBSE_MUTEX; - } - } - else - { - rc = -1; - log_err(rc, __func__, "Uninitialized mutex pass to pthread_mutex_unlock!"); - } - - if (err_msg != NULL) - free(err_msg); - - return rc; - } + return rc; + } void log_record( @@ -586,6 +525,7 @@ void job_array::update_array_values( int job_exit_status) { + updated_array_values++; } void job_array::mark_deleted() {} @@ -606,3 +546,4 @@ batch_request::~batch_request() batch_request::batch_request() {} batch_request::batch_request(int type) : rq_type(type) {} + diff --git a/src/test/req_delete/test_uut.c b/src/test/req_delete/test_uut.c index 907b1a664b..bacbb569a4 100644 --- a/src/test/req_delete/test_uut.c +++ b/src/test/req_delete/test_uut.c @@ -30,6 +30,7 @@ int forced_jobpurge(job *pjob, batch_request *preq); void post_delete_mom2(struct work_task *pwt); int handle_delete_all(batch_request *preq, char *Msg); int handle_single_delete(batch_request *preq, char *Msg); +int perform_job_delete_array_bookkeeping(job *pjob, int cancel_exit_code); bool exit_called; extern int depend_term_called; extern long keep_seconds; @@ -39,6 +40,8 @@ extern int signal_issued; extern int nanny; extern bool br_freed; extern int alloc_work; +extern int updated_array_values; +extern bool find_job_fail; struct server server; extern const char *delpurgestr; @@ -75,6 +78,23 @@ int set_pbs_server_name() return 0; } +START_TEST(test_perform_job_delete_array_bookkeeping) + { + job *pjob = (job *)calloc(1, sizeof(job)); + sprintf(pjob->ji_arraystructid, "1[].roshar"); + sprintf(pjob->ji_qs.ji_jobid, "1[0].roshar"); + + find_job_fail = false; + updated_array_values = 0; + fail_unless(perform_job_delete_array_bookkeeping(pjob, 1) == PBSE_NONE); + fail_unless(updated_array_values == 1); + + find_job_fail = true; + fail_unless(perform_job_delete_array_bookkeeping(pjob, 1) != PBSE_NONE); + fail_unless(updated_array_values == 2); + } +END_TEST + START_TEST(test_handle_single_delete) { batch_request *preq = new batch_request(); @@ -369,6 +389,7 @@ Suite *req_delete_suite(void) tc_core = tcase_create("more"); tcase_add_test(tc_core, test_handle_delete_all); tcase_add_test(tc_core, test_handle_single_delete); + tcase_add_test(tc_core, test_perform_job_delete_array_bookkeeping); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/req_register/scaffolding.c b/src/test/req_register/scaffolding.c index 4b39c3adeb..dd4310bcd4 100644 --- a/src/test/req_register/scaffolding.c +++ b/src/test/req_register/scaffolding.c @@ -25,7 +25,7 @@ const char *msg_registerrel = "Dependency on job %s released."; int i = 2; int svr = 2; int is_attr_set; - +int job_aborted; struct batch_request *alloc_br(int type) { @@ -121,6 +121,7 @@ int svr_chk_owner(struct batch_request *preq, job *pjob) int job_abt(struct job **pjobp, const char *text, bool b=false) { *pjobp = NULL; + job_aborted++; return(0); } diff --git a/src/test/req_register/test_uut.c b/src/test/req_register/test_uut.c index a6e3997490..cd3611fac8 100644 --- a/src/test/req_register/test_uut.c +++ b/src/test/req_register/test_uut.c @@ -41,10 +41,11 @@ void removeAfterAnyDependency(const char *pJobID, const char *targetJob); bool job_ids_match(const char *parent, const char *child); -extern char server_name[]; -extern int i; -extern int svr; -extern int is_attr_set; +extern char server_name[]; +extern int i; +extern int svr; +extern int is_attr_set; +extern int job_aborted; char *job1 = (char *)"1.napali"; char *job2 = (char *)"2.napali"; @@ -101,6 +102,43 @@ START_TEST(set_array_depend_holds_test) fail_unless(register_array_depend(pa, preq, JOB_DEPEND_TYPE_AFTEROKARRAY, 10) == PBSE_NONE); pa->ai_qs.num_successful = 12; fail_unless(set_array_depend_holds(pa) == true); + + // Make sure we abort the job when the dependency can't be fulfilled + job_aborted = 0; + pa->ai_qs.deps.clear(); + fail_unless(register_array_depend(pa, preq, JOB_DEPEND_TYPE_AFTEROKARRAY, 1) == PBSE_NONE); + pa->ai_qs.num_successful = 0; + pa->ai_qs.num_failed = 10; + pa->ai_qs.jobs_done = 10; + pa->ai_qs.num_jobs = 10; + fail_unless(set_array_depend_holds(pa) == false); + fail_unless(job_aborted == 1); + + // Make sure we don't abort a job just because the dependency isn't fulfilled + pa->ai_qs.deps.clear(); + fail_unless(register_array_depend(pa, preq, JOB_DEPEND_TYPE_AFTERNOTOKARRAY, 1) == PBSE_NONE); + pa->ai_qs.num_successful = 10; + pa->ai_qs.num_failed = 0; + pa->ai_qs.jobs_done = 10; + pa->ai_qs.num_jobs = 20; + fail_unless(set_array_depend_holds(pa) == false); + fail_unless(job_aborted == 1); // Still 1 + + // Now abort it + pa->ai_qs.num_successful = 20; + pa->ai_qs.jobs_done = 20; + fail_unless(set_array_depend_holds(pa) == false); + fail_unless(job_aborted == 2); + + // Abort afterstart as well + pa->ai_qs.deps.clear(); + fail_unless(register_array_depend(pa, preq, JOB_DEPEND_TYPE_AFTERSTARTARRAY, 1) == PBSE_NONE); + pa->ai_qs.num_successful = 0; + pa->ai_qs.num_failed = 10; + pa->ai_qs.jobs_done = 10; + pa->ai_qs.num_jobs = 10; + fail_unless(set_array_depend_holds(pa) == false); + fail_unless(job_aborted == 3); } END_TEST From 219382eeecd8a5f196e7523ee81a53b518d0332b Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 9 Dec 2016 12:20:42 -0700 Subject: [PATCH 095/312] TRQ-3847 - fix regression issue with pbsdsh (no output) --- src/cmds/qsub_functions.c | 14 +++++++-- src/lib/Libnet/net_common.c | 33 ++++++++++++++------- src/lib/Libnet/net_server.c | 22 ++++++-------- src/lib/Libnet/port_forwarding.c | 41 +++++++++++++-------------- src/lib/Libnet/rm.c | 25 ++++++++-------- src/resmom/mom_comm.c | 16 ++--------- src/resmom/pbs_demux.c | 12 +------- src/resmom/start_exec.c | 19 +++++++++---- src/test/net_common/scaffolding.c | 2 +- src/test/net_server/scaffolding.c | 6 ++-- src/test/net_server/test_uut.c | 4 +-- src/test/qsub_functions/scaffolding.c | 2 +- src/test/start_exec/scaffolding.c | 6 ++-- src/test/start_exec/test_uut.c | 4 +-- 14 files changed, 104 insertions(+), 102 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 5a65d34659..5f06ccb9cd 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -38,6 +38,10 @@ #include #include #include + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif #include #ifdef sun @@ -2181,15 +2185,19 @@ int wait_for_read_ready( int timeout_sec) { - struct pollfd PollArray; - int n; + struct pollfd PollArray; + struct timespec ts; + int n; PollArray.fd = fd; PollArray.events = POLLIN; PollArray.revents = 0; + ts.tv_sec = timeout_sec; + ts.tv_nsec = 0; + // wait for data ready to read - n = poll(&PollArray, 1, timeout_sec * 1000); + n = ppoll(&PollArray, 1, &ts, NULL); if (n > 0) { diff --git a/src/lib/Libnet/net_common.c b/src/lib/Libnet/net_common.c index b78e2c6331..c6006f61c6 100644 --- a/src/lib/Libnet/net_common.c +++ b/src/lib/Libnet/net_common.c @@ -20,7 +20,12 @@ #include /* errno */ #include /* fcntl, F_GETFL */ #include /* gettimeofday */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif #include /* poll functionality */ + #include #include #include "../lib/Liblog/pbs_log.h" /* log_err */ @@ -516,17 +521,21 @@ int socket_wait_for_write( int socket) { - int rc = PBSE_NONE; - int write_soc = 0; - int sock_errno = 0; - socklen_t len = sizeof(int); - struct pollfd PollArray; + int rc = PBSE_NONE; + int write_soc = 0; + int sock_errno = 0; + socklen_t len = sizeof(int); + struct pollfd PollArray; + struct timespec ts; PollArray.fd = socket; PollArray.events = POLLOUT; PollArray.revents = 0; - if ((write_soc = poll(&PollArray, 1, pbs_tcp_timeout * 1000)) != 1) + ts.tv_sec = pbs_tcp_timeout; + ts.tv_nsec = 0; + + if ((write_soc = ppoll(&PollArray, 1, &ts, NULL)) != 1) { /* timeout is now seen as a permanent failure */ rc = PERMANENT_SOCKET_FAIL; @@ -580,15 +589,19 @@ int socket_wait_for_read( unsigned int timeout) // seconds { - int rc = PBSE_NONE; - int ret; - struct pollfd pfd; + int rc = PBSE_NONE; + int ret; + struct pollfd pfd; + struct timespec ts; pfd.fd = socket; pfd.events = POLLIN | POLLHUP; /* | POLLRDNORM; */ pfd.revents = 0; - ret = poll(&pfd, 1, timeout * 1000); /* poll's timeout is in milliseconds */ + ts.tv_sec = timeout; + ts.tv_nsec = 0; + + ret = ppoll(&pfd, 1, &ts, NULL); if (ret > 0) { char buf[8]; diff --git a/src/lib/Libnet/net_server.c b/src/lib/Libnet/net_server.c index 618e139058..9f4c2c73c1 100644 --- a/src/lib/Libnet/net_server.c +++ b/src/lib/Libnet/net_server.c @@ -102,6 +102,9 @@ #include #include /* thread_func functions */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif #include #if defined(NTOHL_NEEDS_ARPA_INET_H) && defined(HAVE_ARPA_INET_H) #include @@ -335,19 +338,12 @@ int init_network( /* initialize global "read" socket array */ MaxNumDescriptors = get_max_num_descriptors(); - GlobalSocketReadArray = (struct pollfd *)malloc(MaxNumDescriptors * sizeof(struct pollfd)); + GlobalSocketReadArray = (struct pollfd *)calloc(MaxNumDescriptors, sizeof(struct pollfd)); if (GlobalSocketReadArray == NULL) { return(-1); // no memory } - for (i = 0; i < MaxNumDescriptors; i++) - { - GlobalSocketReadArray[i].fd = -1; - GlobalSocketReadArray[i].events = 0; - GlobalSocketReadArray[i].revents = 0; - } - GlobalSocketAddrSet = (u_long *)calloc(MaxNumDescriptors, sizeof(ulong)); if (GlobalSocketAddrSet == NULL) { @@ -691,6 +687,7 @@ int wait_request( time_t now; struct pollfd *PollArray = NULL; + struct timespec ts; int PollArraySize = 0; int MaxNumDescriptors = 0; int SetSize = 0; @@ -699,14 +696,14 @@ int wait_request( char tmpLine[1024]; - int timeout_ms; long OrigState = 0; if (SState != NULL) OrigState = *SState; - // scale seconds to milliseconds - timeout_ms = waittime * 1000; + // set timeout for ppoll() + ts.tv_sec = waittime; + ts.tv_nsec = 0; MaxNumDescriptors = get_max_num_descriptors(); @@ -749,7 +746,7 @@ int wait_request( pthread_mutex_unlock(global_sock_read_mutex); - n = poll(PollArray, MaxNumDescriptors, timeout_ms); + n = ppoll(PollArray, MaxNumDescriptors, &ts, NULL); if (n == -1) { @@ -1028,7 +1025,6 @@ void globalset_del_sock( { pthread_mutex_lock(global_sock_read_mutex); - GlobalSocketReadArray[sock].fd = -1; GlobalSocketReadArray[sock].events = 0; GlobalSocketReadArray[sock].revents = 0; GlobalSocketAddrSet[sock] = 0; diff --git a/src/lib/Libnet/port_forwarding.c b/src/lib/Libnet/port_forwarding.c index 0392d7b621..115478dda8 100644 --- a/src/lib/Libnet/port_forwarding.c +++ b/src/lib/Libnet/port_forwarding.c @@ -40,13 +40,14 @@ void port_forwarder( char *EMsg) /* O */ { - struct pollfd *PollArray; - int rc; - int i; - + int n; + int n2; + int sock; + int num_events; + int rc; struct sockaddr_in from; - torque_socklen_t fromlen; - int n, n2, sock; + torque_socklen_t fromlen; + struct pollfd *PollArray; fromlen = sizeof(from); @@ -59,16 +60,12 @@ void port_forwarder( while (1) { - // clear the array - for (i = 0; i < NUM_SOCKS; i++) - { - PollArray[i].fd = -1; - PollArray[i].events = 0; - PollArray[i].revents = 0; - } - for (n = 0; n < NUM_SOCKS; n++) { + // clear the entry + PollArray[n].events = 0; + PollArray[n].revents = 0; + if (!(socks + n)->active) continue; @@ -76,7 +73,6 @@ void port_forwarder( { PollArray[n].fd = (socks + n)->sock; PollArray[n].events = POLLIN; - PollArray[n].revents = 0; } else { @@ -84,32 +80,33 @@ void port_forwarder( { PollArray[n].fd = (socks + n)->sock; PollArray[n].events = POLLIN; - PollArray[n].revents = 0; } if ((socks + ((socks + n)->peer))->bufavail - (socks + ((socks + n)->peer))->bufwritten > 0) { PollArray[n].fd = (socks + n)->sock; PollArray[n].events |= POLLOUT; - PollArray[n].revents = 0; } } } - rc = poll(PollArray, NUM_SOCKS, -1); + num_events = poll(PollArray, NUM_SOCKS, -1); - if ((rc == -1) && (errno == EINTR)) + if ((num_events == -1) && (errno == EINTR)) continue; - if (rc < 0) + if (num_events < 0) { perror("port forwarding poll()"); exit(EXIT_FAILURE); } - for (n = 0; n < NUM_SOCKS; n++) + for (n = 0; (num_events > 0) && (n < NUM_SOCKS); n++) { + // decrement the count of events returned + num_events--; + if (!(socks + n)->active) continue; @@ -200,7 +197,7 @@ void port_forwarder( } /* END if write */ } /* END foreach fd */ - for (n2 = 0; n2 <= 1;n2++) + for (n2 = 0; n2 <= 1; n2++) { for (n = 0; n < NUM_SOCKS; n++) { diff --git a/src/lib/Libnet/rm.c b/src/lib/Libnet/rm.c index 3ad0b2afd7..ae6588cebb 100644 --- a/src/lib/Libnet/rm.c +++ b/src/lib/Libnet/rm.c @@ -1048,9 +1048,9 @@ int activereq(void) static char id[] = "activereq"; #endif - int i, num; + int i; + int num; int PollArraySize; - int PollArrayIndex = 0; struct pollfd *PollArray; @@ -1067,21 +1067,20 @@ int activereq(void) return(-1); } - for (i = 0; (i < HASHOUT) && (PollArrayIndex < PollArraySize); i++) + // now include the sockets to read + for (i = 0; i < HASHOUT; i++) { struct out *op; - for (op = outs[i]; (op != NULL) && (PollArrayIndex < PollArraySize); op = op->next) + for (op = outs[i]; op != NULL; op = op->next) { - PollArray[PollArrayIndex].fd = op->chan->sock; - PollArray[PollArrayIndex].events = POLLIN; - PollArray[PollArrayIndex].revents = 0; - PollArrayIndex++; + PollArray[op->chan->sock].fd = op->chan->sock; + PollArray[op->chan->sock].events = POLLIN; } } // poll with 15sec timeout - num = poll(PollArray, PollArrayIndex, 15000); + num = poll(PollArray, PollArraySize, 15000); if (num == -1) { @@ -1095,15 +1094,17 @@ int activereq(void) return -2; } - for (i = 0; i < PollArrayIndex; i++) + for (i = 0; (num > 0) && (i < PollArraySize); i++) { + // decrement count of structures with return events + num--; + // something to read? if ((PollArray[i].revents & POLLIN)) { // return socket id that is ready for reading - int sock = PollArray[i].fd; free(PollArray); - return(sock); + return(i); } } diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index f1aa0c9cd4..be1c6f024e 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -8862,7 +8862,6 @@ int readit( close(fd); // remove sock from readset - readset[sock].fd = -1; readset[sock].events = 0; readset[sock].revents = 0; } @@ -8872,7 +8871,6 @@ int readit( close(sock); // remove sock from readset - readset[sock].fd = -1; readset[sock].events = 0; readset[sock].revents = 0; } @@ -9017,28 +9015,18 @@ void fork_demux( pollset_size_bytes = maxfd * sizeof(struct pollfd); - readset = (struct pollfd *)malloc(pollset_size_bytes); + readset = (struct pollfd *)calloc(maxfd, sizeof(struct pollfd)); if (readset == NULL) { - perror("failed to malloc memory for readset"); + perror("failed to calloc memory for readset"); _exit(5); } - // initialize readset - for (i = 0; i < maxfd; i++) - { - readset[i].fd = -1; - readset[i].events = 0; - readset[i].revents = 0; - } - readset[im_mom_stdout].fd = im_mom_stdout; readset[im_mom_stdout].events = POLLIN; - readset[im_mom_stdout].revents = 0; readset[im_mom_stderr].fd = im_mom_stderr; readset[im_mom_stderr].events = POLLIN; - readset[im_mom_stderr].revents = 0; pollset = (struct pollfd *)malloc(pollset_size_bytes); if (pollset == NULL) diff --git a/src/resmom/pbs_demux.c b/src/resmom/pbs_demux.c index 5ffd6df71e..ea3cb3f295 100644 --- a/src/resmom/pbs_demux.c +++ b/src/resmom/pbs_demux.c @@ -269,7 +269,7 @@ int main( pollset_size_bytes = maxfd * sizeof(struct pollfd); - readset = (struct pollfd *)malloc(pollset_size_bytes); + readset = (struct pollfd *)calloc(maxfd, sizeof(struct pollfd)); if (readset == NULL) { perror("cannot alloc memory for readset"); @@ -277,21 +277,11 @@ int main( exit(5); } - // initialize readset - for (i = 0; i < maxfd; i++) - { - readset[i].fd = -1; - readset[i].events = 0; - readset[i].revents = 0; - } - readset[main_sock_out].fd = main_sock_out; readset[main_sock_out].events = POLLIN; - readset[main_sock_out].revents = 0; readset[main_sock_err].fd = main_sock_err; readset[main_sock_err].events = POLLIN; - readset[main_sock_err].revents = 0; // allocate local pollset to hold a copy of readset pollset = (struct pollfd *)malloc(pollset_size_bytes); diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index 582392507a..1721dfb1f0 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -88,7 +88,12 @@ extern "C" #include #include #include + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif #include + #include #include #include @@ -9212,10 +9217,11 @@ int TMomCheckJobChild( int *RC) /* O (return code/errno) */ { - int i; - struct pollfd PollArray; - int rc; - int read_size = sizeof(struct startjob_rtn); + int i; + struct pollfd PollArray; + struct timespec ts; + int rc; + int read_size = sizeof(struct startjob_rtn); /* NOTE: assume if anything is on pipe, everything is on pipe (may reasult in hang) */ @@ -9231,7 +9237,10 @@ int TMomCheckJobChild( PollArray.events = POLLIN; PollArray.revents = 0; - rc = poll(&PollArray, 1, Timeout * 1000); + ts.tv_sec = Timeout; + ts.tv_nsec = 0; + + rc = ppoll(&PollArray, 1, &ts, NULL); if ((rc <= 0) || ((PollArray.revents & POLLIN) == 0)) { diff --git a/src/test/net_common/scaffolding.c b/src/test/net_common/scaffolding.c index f17299b9c9..e8dce86c96 100644 --- a/src/test/net_common/scaffolding.c +++ b/src/test/net_common/scaffolding.c @@ -84,7 +84,7 @@ int connect( return(-1); } -int poll(struct pollfd *fds, nfds_t nfds, int timeout) +int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const sigset_t *sigmask) { fds->revents = global_poll_revents; diff --git a/src/test/net_server/scaffolding.c b/src/test/net_server/scaffolding.c index cbc4c6d371..e5611732f4 100644 --- a/src/test/net_server/scaffolding.c +++ b/src/test/net_server/scaffolding.c @@ -17,7 +17,7 @@ bool socket_read_success = true; bool socket_read_code = true; nfds_t global_poll_nfds = 0; -int global_poll_timeout_ms = 0; +int global_poll_timeout_sec = 0; const char *msg_daemonname = "unset"; @@ -152,9 +152,9 @@ int get_max_num_descriptors(void) return(getdtablesize()); } -int poll(struct pollfd *fds, nfds_t nfds, int timeout) +int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const sigset_t *sigmask) { global_poll_nfds = nfds; - global_poll_timeout_ms = timeout; + global_poll_timeout_sec = timeout->tv_sec; return(0); } diff --git a/src/test/net_server/test_uut.c b/src/test/net_server/test_uut.c index 4fc8398f89..a7ce05787d 100644 --- a/src/test/net_server/test_uut.c +++ b/src/test/net_server/test_uut.c @@ -23,7 +23,7 @@ extern bool socket_read_success; extern bool socket_read_code; extern nfds_t global_poll_nfds; -extern int global_poll_timeout_ms; +extern int global_poll_timeout_sec; int add_connection(int sock, enum conn_type type, pbs_net_t addr, unsigned int port, unsigned int socktype, void *(*func)(void *), int add_wait_request); void *accept_conn(void *new_conn); @@ -166,7 +166,7 @@ START_TEST(test_init_network) // examine the values passed to poll() to see if they were expected fail_unless(global_poll_nfds == MaxNumDescriptors); - fail_unless(global_poll_timeout_ms == timeout_sec * 1000); + fail_unless(global_poll_timeout_sec == timeout_sec); } END_TEST diff --git a/src/test/qsub_functions/scaffolding.c b/src/test/qsub_functions/scaffolding.c index 9e1d24df43..0becf9ebbd 100644 --- a/src/test/qsub_functions/scaffolding.c +++ b/src/test/qsub_functions/scaffolding.c @@ -485,7 +485,7 @@ struct group *getgrnam_ext( return(grp); } /* END getgrnam_ext() */ -int poll(struct pollfd *fds, nfds_t nfds, int timeout) +int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const sigset_t *sigmask) { fds->revents = global_poll_revents; errno = global_poll_errno; diff --git a/src/test/start_exec/scaffolding.c b/src/test/start_exec/scaffolding.c index ce2a2a5f48..b5a2222e1b 100644 --- a/src/test/start_exec/scaffolding.c +++ b/src/test/start_exec/scaffolding.c @@ -106,7 +106,7 @@ int task_saved; std::string presetup_prologue; int global_poll_fd = -1; -int global_poll_timeout_ms = -1; +int global_poll_timeout_sec = -1; #ifdef NUMA_SUPPORT nodeboard node_boards[MAX_NODE_BOARDS]; @@ -1106,12 +1106,12 @@ int setup_gpus_for_job(job *pjob) {return(0);} -int poll(struct pollfd *fds, nfds_t nfds, int timeout) +int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const sigset_t *sigmask) { if (nfds > 0) { global_poll_fd = fds[0].fd; - global_poll_timeout_ms = timeout; + global_poll_timeout_sec = timeout->tv_sec; } return(0); diff --git a/src/test/start_exec/test_uut.c b/src/test/start_exec/test_uut.c index 3e45eb7b16..d985c71eb8 100644 --- a/src/test/start_exec/test_uut.c +++ b/src/test/start_exec/test_uut.c @@ -46,7 +46,7 @@ extern bool am_ms; extern bool addr_fail; extern int global_poll_fd; -extern int global_poll_timeout_ms; +extern int global_poll_timeout_sec; void create_command(std::string &cmd, char **argv); void no_hang(int sig); @@ -778,7 +778,7 @@ START_TEST(test_TMomCheckJobChild) TJE.jsmpipe[0] = fd; TMomCheckJobChild(&TJE, timeout_sec, &count, &rc); fail_unless(global_poll_fd == fd); - fail_unless(global_poll_timeout_ms == timeout_sec * 1000); + fail_unless(global_poll_timeout_sec == timeout_sec); } END_TEST From 2034211b16160093c15d62c0a8dda9687c66a4f0 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 9 Dec 2016 16:00:28 -0700 Subject: [PATCH 096/312] TRQ-3847 - fix interactive issue with pbsdsh (no output) --- src/lib/Libnet/net_server.c | 84 ++++++++++++++++---------------- src/lib/Libnet/port_forwarding.c | 4 ++ src/lib/Libnet/rm.c | 4 ++ src/resmom/mom_comm.c | 4 ++ src/resmom/pbs_demux.c | 4 +- 5 files changed, 57 insertions(+), 43 deletions(-) diff --git a/src/lib/Libnet/net_server.c b/src/lib/Libnet/net_server.c index 9f4c2c73c1..d583e3f369 100644 --- a/src/lib/Libnet/net_server.c +++ b/src/lib/Libnet/net_server.c @@ -790,62 +790,62 @@ int wait_request( } /* END else (errno == EINTR) */ } /* END if (n == -1) */ - for (i = 0; (i < max_connection) && (n > 0); i++) + for (i = 0; (n > 0) && (i < max_connection); i++) { - // any events? - if (PollArray[i].revents != 0) - { - // decrement the count of structures that have non-zero revents - n--; - - // is there data ready to read? - if ((PollArray[i].revents & POLLIN)) - { - pthread_mutex_lock(svr_conn[i].cn_mutex); - /* this socket has data */ + // skip entry if it has no return events + if (PollArray[i].revents == 0) + continue; - svr_conn[i].cn_lasttime = time(NULL); + // decrement the count of structures that have non-zero revents + n--; - if (svr_conn[i].cn_active != Idle) - { - void *(*func)(void *) = svr_conn[i].cn_func; + // is there data ready to read? + if ((PollArray[i].revents & POLLIN)) + { + pthread_mutex_lock(svr_conn[i].cn_mutex); + /* this socket has data */ - netcounter_incr(); + svr_conn[i].cn_lasttime = time(NULL); - pthread_mutex_unlock(svr_conn[i].cn_mutex); + if (svr_conn[i].cn_active != Idle) + { + void *(*func)(void *) = svr_conn[i].cn_func; - if (func != NULL) - { - int args[3]; + netcounter_incr(); - args[0] = i; - args[1] = (int)SocketAddrSet[i]; - args[2] = (int)SocketPortSet[i]; - func((void *)args); - } + pthread_mutex_unlock(svr_conn[i].cn_mutex); - /* NOTE: breakout if state changed (probably received shutdown request) */ + if (func != NULL) + { + int args[3]; - if ((SState != NULL) && - (OrigState != *SState)) - break; + args[0] = i; + args[1] = (int)SocketAddrSet[i]; + args[2] = (int)SocketPortSet[i]; + func((void *)args); } - else - { - pthread_mutex_unlock(svr_conn[i].cn_mutex); - globalset_del_sock(i); - close_conn(i, FALSE); + /* NOTE: breakout if state changed (probably received shutdown request) */ + + if ((SState != NULL) && + (OrigState != *SState)) + break; + } + else + { + pthread_mutex_unlock(svr_conn[i].cn_mutex); + + globalset_del_sock(i); + close_conn(i, FALSE); - pthread_mutex_lock(num_connections_mutex); + pthread_mutex_lock(num_connections_mutex); - sprintf(tmpLine, "closed connections to fd %d - num_connections=%d (poll bad socket)", - i, - num_connections); + sprintf(tmpLine, "closed connections to fd %d - num_connections=%d (poll bad socket)", + i, + num_connections); - pthread_mutex_unlock(num_connections_mutex); - log_err(-1, __func__, tmpLine); - } + pthread_mutex_unlock(num_connections_mutex); + log_err(-1, __func__, tmpLine); } } } /* END for i */ diff --git a/src/lib/Libnet/port_forwarding.c b/src/lib/Libnet/port_forwarding.c index 115478dda8..55c139ae4e 100644 --- a/src/lib/Libnet/port_forwarding.c +++ b/src/lib/Libnet/port_forwarding.c @@ -104,6 +104,10 @@ void port_forwarder( for (n = 0; (num_events > 0) && (n < NUM_SOCKS); n++) { + // skip entry with no return events + if (PollArray[n].revents == 0) + continue; + // decrement the count of events returned num_events--; diff --git a/src/lib/Libnet/rm.c b/src/lib/Libnet/rm.c index ae6588cebb..09953380fa 100644 --- a/src/lib/Libnet/rm.c +++ b/src/lib/Libnet/rm.c @@ -1096,6 +1096,10 @@ int activereq(void) for (i = 0; (num > 0) && (i < PollArraySize); i++) { + // skip entry with no return events + if (PollArray[i].revents == 0) + continue; + // decrement count of structures with return events num--; diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index be1c6f024e..5d85a95652 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -9153,6 +9153,10 @@ void fork_demux( for (i = 0; (n > 0) && (i < maxfd); i++) { + // skip entry with no return events + if (pollset[i].revents == 0) + continue; + // decrement count of structures that have return events n--; diff --git a/src/resmom/pbs_demux.c b/src/resmom/pbs_demux.c index ea3cb3f295..63745ccfd5 100644 --- a/src/resmom/pbs_demux.c +++ b/src/resmom/pbs_demux.c @@ -172,7 +172,6 @@ void readit( prm->r_where = invalid; // remove socket from readset - readset[sock].fd = -1; readset[sock].events = 0; readset[sock].revents = 0; } @@ -345,6 +344,9 @@ int main( for (i = 0; (n > 0) && (i < maxfd); i++) { + if (pollset[i].revents == 0) + continue; + // decrement count of structures with non-zero return events n--; From 473076db5fae0079abe0d997ba47b58e07f54de4 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 9 Dec 2016 15:07:57 -0700 Subject: [PATCH 097/312] Fix two Coverity bugs. --- src/server/req_delete.c | 29 ++++++++++++++++------------- src/server/req_register.c | 1 + 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/server/req_delete.c b/src/server/req_delete.c index d9cb3ab012..8ce5c62f58 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -344,20 +344,23 @@ int perform_job_delete_array_bookkeeping( if (pjob == NULL) return(PBSE_JOBNOTFOUND); - int old_state = pjob->ji_qs.ji_state; - std::string jobid(pjob->ji_qs.ji_jobid); - - unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); - - pa->update_array_values(old_state, - aeTerminate, - jobid.c_str(), - cancel_exit_code); - - if ((pjob = svr_find_job((char *)jobid.c_str(),FALSE)) == NULL) - rc = PBSE_JOBNOTFOUND; + if (pa != NULL) + { + int old_state = pjob->ji_qs.ji_state; + std::string jobid(pjob->ji_qs.ji_jobid); - unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); + unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); + + pa->update_array_values(old_state, + aeTerminate, + jobid.c_str(), + cancel_exit_code); + + if ((pjob = svr_find_job((char *)jobid.c_str(),FALSE)) == NULL) + rc = PBSE_JOBNOTFOUND; + + unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); + } } return(rc); diff --git a/src/server/req_register.c b/src/server/req_register.c index cd2a9411f2..4257863e21 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -1144,6 +1144,7 @@ bool set_array_depend_holds( { array_depend *pdep = *it; + could_fulfill_dependency = pa->ai_qs.array_size; compare_number = -1; switch (pdep->dp_type) From f14a108f0c4660b512e4d7f21745d287c6070e81 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 12 Dec 2016 15:14:47 -0700 Subject: [PATCH 098/312] TRQ-3879. Fix another crash on a rare failure type. --- src/resmom/mom_comm.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index 5d85a95652..d295cbeb32 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -2092,7 +2092,8 @@ void send_im_error( int rc = DIS_SUCCESS; struct tcp_chan *local_chan = NULL; - if (reply) + if ((reply) && + (pjob->ji_hosts != NULL)) { for (i = 0; i < 5; i++) { @@ -2156,6 +2157,13 @@ void send_im_error( log_err(-1, __func__, log_buffer); } + } + else if (pjob->ji_hosts == NULL) + { + sprintf(log_buffer, + "Cannot send error reply on event %d for job %s because the hosts weren't correctly decoded.", + event, pjob->ji_qs.ji_jobid); + log_err(-1, __func__, log_buffer); } } /* END send_im_error() */ From 1526925314ad4395a3938ec0bf79791a9bcf33d1 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 12 Dec 2016 16:24:56 -0700 Subject: [PATCH 099/312] TRQ-3847 - initialize all poll fd entries to -1 --- src/lib/Libnet/net_server.c | 12 +++++++++++- src/lib/Libnet/port_forwarding.c | 1 + src/lib/Libnet/rm.c | 12 ++++++++++-- src/resmom/mom_comm.c | 14 ++++++++++++-- src/resmom/pbs_demux.c | 11 ++++++++++- 5 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/lib/Libnet/net_server.c b/src/lib/Libnet/net_server.c index d583e3f369..4de9877c32 100644 --- a/src/lib/Libnet/net_server.c +++ b/src/lib/Libnet/net_server.c @@ -338,12 +338,20 @@ int init_network( /* initialize global "read" socket array */ MaxNumDescriptors = get_max_num_descriptors(); - GlobalSocketReadArray = (struct pollfd *)calloc(MaxNumDescriptors, sizeof(struct pollfd)); + GlobalSocketReadArray = (struct pollfd *)malloc(MaxNumDescriptors * sizeof(struct pollfd)); if (GlobalSocketReadArray == NULL) { return(-1); // no memory } + // set initial values + for (i = 0; i < MaxNumDescriptors; i++) + { + GlobalSocketReadArray[i].fd = -1; + GlobalSocketReadArray[i].events = 0; + GlobalSocketReadArray[i].revents = 0; + } + GlobalSocketAddrSet = (u_long *)calloc(MaxNumDescriptors, sizeof(ulong)); if (GlobalSocketAddrSet == NULL) { @@ -1011,6 +1019,7 @@ void globalset_add_sock( pthread_mutex_lock(global_sock_read_mutex); GlobalSocketReadArray[sock].fd = sock; GlobalSocketReadArray[sock].events = POLLIN; + GlobalSocketReadArray[sock].revents = 0; GlobalSocketAddrSet[sock] = addr; GlobalSocketPortSet[sock] = port; pthread_mutex_unlock(global_sock_read_mutex); @@ -1025,6 +1034,7 @@ void globalset_del_sock( { pthread_mutex_lock(global_sock_read_mutex); + GlobalSocketReadArray[sock].fd = -1; GlobalSocketReadArray[sock].events = 0; GlobalSocketReadArray[sock].revents = 0; GlobalSocketAddrSet[sock] = 0; diff --git a/src/lib/Libnet/port_forwarding.c b/src/lib/Libnet/port_forwarding.c index 55c139ae4e..e07d11f515 100644 --- a/src/lib/Libnet/port_forwarding.c +++ b/src/lib/Libnet/port_forwarding.c @@ -63,6 +63,7 @@ void port_forwarder( for (n = 0; n < NUM_SOCKS; n++) { // clear the entry + PollArray[n].fd = -1; PollArray[n].events = 0; PollArray[n].revents = 0; diff --git a/src/lib/Libnet/rm.c b/src/lib/Libnet/rm.c index 09953380fa..b59c4cd522 100644 --- a/src/lib/Libnet/rm.c +++ b/src/lib/Libnet/rm.c @@ -1058,15 +1058,23 @@ int activereq(void) // initialize the poll array PollArraySize = get_max_num_descriptors(); - PollArray = (struct pollfd *)calloc(PollArraySize, sizeof(struct pollfd)); + PollArray = (struct pollfd *)malloc(PollArraySize * sizeof(struct pollfd)); if (PollArray == NULL) { // no memory - DBPRT(("%s: calloc %d %s\n", id, errno, pbs_strerror(errno))) + DBPRT(("%s: malloc %d %s\n", id, errno, pbs_strerror(errno))) return(-1); } + // set initial values + for (i = 0; i < PollArraySize; i++) + { + PollArray[i].fd = -1; + PollArray[i].events = 0; + PollArray[i].revents = 0; + } + // now include the sockets to read for (i = 0; i < HASHOUT; i++) { diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index d295cbeb32..14a4bf11e5 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -8870,6 +8870,7 @@ int readit( close(fd); // remove sock from readset + readset[sock].fd = -1; readset[sock].events = 0; readset[sock].revents = 0; } @@ -8879,6 +8880,7 @@ int readit( close(sock); // remove sock from readset + readset[sock].fd = -1; readset[sock].events = 0; readset[sock].revents = 0; } @@ -9023,13 +9025,21 @@ void fork_demux( pollset_size_bytes = maxfd * sizeof(struct pollfd); - readset = (struct pollfd *)calloc(maxfd, sizeof(struct pollfd)); + readset = (struct pollfd *)malloc(maxfd * sizeof(struct pollfd)); if (readset == NULL) { - perror("failed to calloc memory for readset"); + perror("failed to malloc memory for readset"); _exit(5); } + // set initial values + for (i = 0; i < maxfd; i++) + { + readset[i].fd = -1; + readset[i].events = 0; + readset[i].revents = 0; + } + readset[im_mom_stdout].fd = im_mom_stdout; readset[im_mom_stdout].events = POLLIN; diff --git a/src/resmom/pbs_demux.c b/src/resmom/pbs_demux.c index 63745ccfd5..e3de06577e 100644 --- a/src/resmom/pbs_demux.c +++ b/src/resmom/pbs_demux.c @@ -172,6 +172,7 @@ void readit( prm->r_where = invalid; // remove socket from readset + readset[sock].fd = -1; readset[sock].events = 0; readset[sock].revents = 0; } @@ -268,7 +269,7 @@ int main( pollset_size_bytes = maxfd * sizeof(struct pollfd); - readset = (struct pollfd *)calloc(maxfd, sizeof(struct pollfd)); + readset = (struct pollfd *)malloc(maxfd * sizeof(struct pollfd)); if (readset == NULL) { perror("cannot alloc memory for readset"); @@ -276,6 +277,14 @@ int main( exit(5); } + // set initial values + for (i = 0; i < maxfd; i++) + { + readset[i].fd = -1; + readset[i].events = 0; + readset[i].revents = 0; + } + readset[main_sock_out].fd = main_sock_out; readset[main_sock_out].events = POLLIN; From 217431e1cc0f6e5377a20fd66b57bd94fe33085a Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 12 Dec 2016 17:40:28 -0700 Subject: [PATCH 100/312] TRQ-3832. Get the amount of file space available to an unprivileged user by default. --- src/resmom/linux/mom_mach.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/resmom/linux/mom_mach.c b/src/resmom/linux/mom_mach.c index 837f4e3bbd..6e53b2cf6f 100644 --- a/src/resmom/linux/mom_mach.c +++ b/src/resmom/linux/mom_mach.c @@ -4729,10 +4729,10 @@ char *size_fs( return(NULL); } -#ifdef RPT_BAVAIL -#define RPT_STATFS_MEMBER f_bavail -#else +#ifdef RPT_BFREE #define RPT_STATFS_MEMBER f_bfree +#else +#define RPT_STATFS_MEMBER f_bavail #endif sprintf(ret_string, "%lukb:%lukb", From fb55b68b1264310142cc322309f3203161768293 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 13 Dec 2016 15:01:38 -0700 Subject: [PATCH 101/312] Fix a unit test issue. --- src/test/numa_core/test_uut.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/numa_core/test_uut.c b/src/test/numa_core/test_uut.c index 66a60950bc..877e7da4b6 100644 --- a/src/test/numa_core/test_uut.c +++ b/src/test/numa_core/test_uut.c @@ -34,7 +34,7 @@ END_TEST START_TEST(test_displayAsString) { Core c; - c.add_processing_unit(CORE, 0); + c.add_processing_unit(CORE_INT, 0); std::stringstream out; c.displayAsString(out); @@ -105,19 +105,19 @@ START_TEST(test_add_processing_unit) fail_unless(c.getNumberOfProcessingUnits() == 0, "%d processing units", c.getNumberOfProcessingUnits()); fail_unless(c.get_id() == -1); - c.add_processing_unit(CORE, 0); - c.add_processing_unit(THREAD, 1); + c.add_processing_unit(CORE_INT, 0); + c.add_processing_unit(THREAD_INT, 1); fail_unless(c.getNumberOfProcessingUnits() == 2); fail_unless(c.get_id() == 0); - fail_unless(c.add_processing_unit(CORE, 2) != 0); + fail_unless(c.add_processing_unit(CORE_INT, 2) != 0); fail_unless(c.getNumberOfProcessingUnits() == 2); fail_unless(c.get_id() == 0); Core c2; - c2.add_processing_unit(CORE, 4); - c2.add_processing_unit(THREAD, 5); - c2.add_processing_unit(THREAD, 6); - c2.add_processing_unit(THREAD, 7); + c2.add_processing_unit(CORE_INT, 4); + c2.add_processing_unit(THREAD_INT, 5); + c2.add_processing_unit(THREAD_INT, 6); + c2.add_processing_unit(THREAD_INT, 7); fail_unless(c2.getNumberOfProcessingUnits() == 4, "%d", c2.getNumberOfProcessingUnits()); fail_unless(c2.get_id() == 4); } From cc748e0a678a9ccc559b30edba63b5373acf7094 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 13 Dec 2016 15:07:04 -0700 Subject: [PATCH 102/312] TRQ-3887. Fix a scenario where two jobs are sent to the same GPU. --- src/server/node_manager.c | 192 ++++++++++--------------------- src/test/node_manager/test_uut.c | 19 +++ 2 files changed, 81 insertions(+), 130 deletions(-) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 6fdf2a4cf3..138b78354e 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -1871,12 +1871,21 @@ static int hasppn( /* -** Count how many gpus are available for use on this node -*/ + * gpu_count() + * + * Get a gpu count for this node + * if freeonly is true, then count the free gpus; if false, then return the number + * of gpus. + * + * @param pnode - the node whose gpus we're counting + * @param freeonly - specifies if we want a total or just the available gpus + * @return the total gpus if freeonly is false, else the number of free gpus + */ + int gpu_count( - struct pbsnode *pnode, /* I */ - int freeonly) /* I */ + pbsnode *pnode, /* I */ + bool freeonly) /* I */ { int count = 0; @@ -1900,42 +1909,10 @@ int gpu_count( return (count); } - if (pnode->nd_gpus_real) - { - int j; - - for (j = 0; j < pnode->nd_ngpus; j++) - { - struct gpusubn &gn = pnode->nd_gpusn[j]; - - /* always ignore unavailable gpus */ - if (gn.state == gpu_unavailable) - continue; - - if (!freeonly) - { - count++; - } - else if ((gn.state == gpu_unallocated) || - ((gn.state == gpu_shared) && - (gpu_mode_rqstd == gpu_normal))) - { - count++;; - } - } - } + if (freeonly == false) + count = pnode->nd_ngpus; else - { - /* virtual gpus */ - if (freeonly) - { - count = pnode->nd_ngpus_free; - } - else - { - count = pnode->nd_ngpus; - } - } + count = pnode->nd_ngpus_free; if (LOGLEVEL >= 7) { @@ -2397,7 +2374,7 @@ bool node_is_spec_acceptable( return(false); if ((hasppn(pnode, ppn_req, SKIP_NONE) == FALSE) || - (gpu_count(pnode, FALSE) < gpu_req) || + (gpu_count(pnode, false) < gpu_req) || (pnode->nd_nmics < mic_req)) return(false); @@ -2406,7 +2383,7 @@ bool node_is_spec_acceptable( if (((pnode->nd_state & (INUSE_OFFLINE | INUSE_NOT_READY | INUSE_RESERVE | INUSE_JOB)) != 0)||(pnode->nd_power_state != POWER_STATE_RUNNING)) return(false); - gpu_free = gpu_count(pnode, TRUE) - pnode->nd_ngpus_to_be_used; + gpu_free = gpu_count(pnode, true) - pnode->nd_ngpus_to_be_used; np_free = pnode->nd_slots.get_number_free() - pnode->nd_np_to_be_used; mic_free = pnode->nd_nmics_free - pnode->nd_nmics_to_be_used; @@ -3704,21 +3681,14 @@ int reserve_node( int add_job_to_gpu_subnode( - struct pbsnode *pnode, - gpusubn &gn, - job *pjob) + pbsnode *pnode, + gpusubn &gn, + job *pjob) { - if (!pnode->nd_gpus_real) - { - /* update the gpu subnode */ - gn.job_internal_id = pjob->ji_internal_id; - gn.inuse = TRUE; - - /* update the main node */ - pnode->nd_ngpus_free--; - } - + gn.job_internal_id = pjob->ji_internal_id; + pnode->nd_ngpus_free--; + gn.inuse = true; gn.job_count++; pnode->nd_ngpus_to_be_used--; @@ -3843,30 +3813,8 @@ int place_gpus_in_hostlist( gpusubn &gn = pnode->nd_gpusn[j]; - if (pnode->nd_gpus_real) - { - if ((gn.state == gpu_unavailable) || - (gn.state == gpu_shared) || - (gn.state == gpu_exclusive) || - ((((int)gn.mode == gpu_normal)) && - (gpu_mode_rqstd != gpu_normal) && - (gn.state != gpu_unallocated))) - continue; - } - else - { - if ((gn.state == gpu_unavailable) || - (gn.inuse == TRUE)) - continue; - } - - if ((gn.state == gpu_unavailable) || - ((gn.state == gpu_exclusive) && pnode->nd_gpus_real) || - ((pnode->nd_gpus_real) && - ((int)gn.mode == gpu_normal) && - ((gpu_mode_rqstd != gpu_normal) && (gn.state != gpu_unallocated))) || - ((!pnode->nd_gpus_real) && - (gn.inuse == TRUE))) + if ((gn.inuse == true) || + (gn.state == gpu_unavailable)) continue; add_job_to_gpu_subnode(pnode,gn,pjob); @@ -5543,17 +5491,23 @@ char *get_next_exec_host( +/* + * remove_job_from_nodes_gpus() + * + * @param pnode - the node whose job is being removed + * @param pjob - the job that should be removed from the gpus + * @return PBSE_NONE + */ + int remove_job_from_nodes_gpus( - struct pbsnode *pnode, - job *pjob) + pbsnode *pnode, + job *pjob) { char *gpu_str = NULL; int i; char log_buf[LOCAL_LOG_BUF_SIZE]; - std::string tmp_str; - char num_str[6]; if (pjob->ji_wattr[JOB_ATR_exec_gpus].at_flags & ATR_VFLAG_SET) gpu_str = pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str; @@ -5565,56 +5519,27 @@ int remove_job_from_nodes_gpus( { gpusubn &gn = pnode->nd_gpusn[i]; - if (pnode->nd_gpus_real) + if (gn.job_internal_id == pjob->ji_internal_id) { - /* reset real gpu nodes */ - tmp_str = pnode->get_name(); - tmp_str += "-gpu/"; - sprintf (num_str, "%d", i); - tmp_str += num_str; - - /* look thru the string and see if it has this host and gpuid. - * exec_gpus string should be in format of - * -gpu/[+-gpu/...] - * - * if we are using the gpu node exclusively or if shared mode and - * this is last job assigned to this gpu then set it's state - * unallocated so its available for a new job. Takes time to get the - * gpu status report from the moms. - */ - - if (strstr(gpu_str, tmp_str.c_str()) != NULL) - { - gn.job_count--; + gn.job_internal_id = -1; + gn.inuse = false; + pnode->nd_ngpus_free++; + gn.job_count--; - if ((gn.mode == gpu_exclusive_thread) || - (gn.mode == gpu_exclusive_process) || - ((gn.mode == gpu_normal) && - (gn.job_count == 0))) - { - gn.state = gpu_unallocated; - - if (LOGLEVEL >= 7) - { - sprintf(log_buf, "freeing node %s gpu %d for job %s", - pnode->get_name(), - i, - pjob->ji_qs.ji_jobid); - - log_record(PBSEVENT_SCHED, PBS_EVENTCLASS_REQUEST, __func__, log_buf); - } - - } - } - } - else - { - if (gn.job_internal_id == pjob->ji_internal_id) + if ((gn.mode == gpu_exclusive_thread) || + (gn.mode == gpu_exclusive_process) || + ((gn.mode == gpu_normal) && + (gn.job_count == 0))) + gn.state = gpu_unallocated; + + if (LOGLEVEL >= 7) { - gn.inuse = FALSE; - gn.job_internal_id = -1; + sprintf(log_buf, "freeing node %s gpu %d for job %s", + pnode->get_name(), + i, + pjob->ji_qs.ji_jobid); - pnode->nd_ngpus_free++; + log_record(PBSEVENT_SCHED, PBS_EVENTCLASS_REQUEST, __func__, log_buf); } } } @@ -5625,11 +5550,18 @@ int remove_job_from_nodes_gpus( +/* + * remove_job_from_node() + * + * @param pnode - the node removeing the job + * @param internal_job_id - the internal job id from the job to be removed + * @return PBSE_NONE + */ int remove_job_from_node( - struct pbsnode *pnode, - int internal_job_id) + pbsnode *pnode, + int internal_job_id) { FUNCTION_TIMER diff --git a/src/test/node_manager/test_uut.c b/src/test/node_manager/test_uut.c index 476dd67440..a17d4bf7dc 100644 --- a/src/test/node_manager/test_uut.c +++ b/src/test/node_manager/test_uut.c @@ -41,6 +41,7 @@ int add_job_to_mic(struct pbsnode *pnode, int index, job *pjob); int remove_job_from_nodes_mics(struct pbsnode *pnode, job *pjob); void update_failure_counts(const char *node_name, int rc); void check_node_jobs_existence(struct work_task *pwt); +int add_job_to_gpu_subnode(pbsnode *pnode, gpusubn &gn, job *pjob); @@ -54,6 +55,23 @@ extern bool alloc_br_success; extern bool cray_enabled; +START_TEST(add_job_to_gpu_subnode_test) + { + gpusubn gn; + job pjob; + pbsnode pnode; + + pnode.nd_ngpus_to_be_used = 1; + pjob.ji_internal_id = 10; + + fail_unless(add_job_to_gpu_subnode(&pnode, gn, &pjob) == PBSE_NONE); + fail_unless(gn.inuse == true); + fail_unless(pnode.nd_ngpus_to_be_used == 0); + fail_unless(gn.job_internal_id == pjob.ji_internal_id); + } +END_TEST + + START_TEST(check_node_jobs_exitence_test) { napali_node.change_name(napali); @@ -931,6 +949,7 @@ Suite *node_manager_suite(void) tc_core = tcase_create("place_subnodes_in_hostlist_job_exclusive_test"); tcase_add_test(tc_core, place_subnodes_in_hostlist_job_exclusive_test); + tcase_add_test(tc_core, add_job_to_gpu_subnode_test); suite_add_tcase(s, tc_core); tc_core = tcase_create("record_external_node_test"); From a06798cdcb49ca7c1165a3e4f1d16bc05a681f2e Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 13 Dec 2016 15:12:44 -0700 Subject: [PATCH 103/312] TRQ-3886. Fix a rare crash in freeing array subjobs. --- src/include/array.h | 1 + src/server/job_array.cpp | 44 ++++++++++++++++++++++++++++++++ src/server/job_func.c | 20 +-------------- src/test/job_array/test_uut.c | 39 +++++++++++++++++++++++++++- src/test/job_func/scaffolding.c | 5 ++++ src/test/job_recov/scaffolding.c | 5 ++++ 6 files changed, 94 insertions(+), 20 deletions(-) diff --git a/src/include/array.h b/src/include/array.h index fdb064e251..89b8676986 100644 --- a/src/include/array.h +++ b/src/include/array.h @@ -175,6 +175,7 @@ class job_array bool need_to_update_slot_limits() const; void mark_deleted(); bool is_deleted() const; + bool mark_end_of_subjob(job *pjob); }; diff --git a/src/server/job_array.cpp b/src/server/job_array.cpp index c19e4a46dd..63f06c9db0 100644 --- a/src/server/job_array.cpp +++ b/src/server/job_array.cpp @@ -570,3 +570,47 @@ bool job_array::is_deleted() const +/* + * mark_end_of_subjob() + * Takes care of noting that this subjob is being purged from the array + * + * @param pjob - the subjob that is being removed + * @return true if the array has no more subjobs, false otherwise + */ + +bool job_array::mark_end_of_subjob( + + job *pjob) + + { + bool no_more_subjobs = false; + int index; + + if (pjob != NULL) + { + if (this->job_ids != NULL) + { + index = pjob->ji_wattr[JOB_ATR_job_array_id].at_val.at_long; + + if ((index >= 0) && + (index < this->ai_qs.num_jobs)) + { + if (this->job_ids[index] != NULL) + { + free(this->job_ids[index]); + this->job_ids[index] = NULL; + + this->ai_qs.num_purged++; + + if ((this->ai_qs.num_purged == this->ai_qs.num_jobs) || + ((this->is_deleted() == true) && + (this->ai_qs.num_idle == 0))) + no_more_subjobs = true; + } + } + } + } + + return(no_more_subjobs); + } // mark_end_of_subjob() + diff --git a/src/server/job_func.c b/src/server/job_func.c index 67fb86a9cb..e7eed7d92a 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -1937,24 +1937,7 @@ int svr_job_purge( { if (pa != NULL) { - if (pa->job_ids != NULL) - { - free(pa->job_ids[pjob->ji_wattr[JOB_ATR_job_array_id].at_val.at_long]); - pa->job_ids[pjob->ji_wattr[JOB_ATR_job_array_id].at_val.at_long] = NULL; - } - - /* if there are no more jobs in the array, - * then we can clean that up too */ - pa->ai_qs.num_purged++; - if ((pa->ai_qs.num_purged == pa->ai_qs.num_jobs) || - ((pa->is_deleted() == true) && - (pa->ai_qs.num_idle == 0))) - { - /* array_delete will unlock pa->ai_mutex */ - strcpy(array_id, pjob->ji_arraystructid); - do_delete_array = true; - } - else + if ((do_delete_array = pa->mark_end_of_subjob(pjob)) == false) array_save(pa); unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); @@ -1967,7 +1950,6 @@ int svr_job_purge( } } - if ((job_substate != JOB_SUBSTATE_TRANSIN) && (job_substate != JOB_SUBSTATE_TRANSICM)) { diff --git a/src/test/job_array/test_uut.c b/src/test/job_array/test_uut.c index 8bb5fc2ff0..8c062a2894 100644 --- a/src/test/job_array/test_uut.c +++ b/src/test/job_array/test_uut.c @@ -2,7 +2,7 @@ #include #include - +#include "pbs_job.h" #include "pbs_error.h" #include "array.h" #include @@ -139,6 +139,42 @@ START_TEST(test_parse_array_request) END_TEST +START_TEST(test_mark_end_of_subjob) + { + job_array pa; + array_size = 1000; + + pa.parse_array_request("0-9"); + for (int i = 0; i < 10; i++) + pa.create_job_if_needed(); + + fail_unless(pa.ai_qs.highest_id_created == 9); + + fail_unless(pa.mark_end_of_subjob(NULL) == false); + + job pjob; + + // Make sure a bad index doesn't crash + pjob.ji_wattr[JOB_ATR_job_array_id].at_val.at_long = -1; + fail_unless(pa.mark_end_of_subjob(&pjob) == false); + + for (int i = 0; i < 10; i++) + { + pjob.ji_wattr[JOB_ATR_job_array_id].at_val.at_long = i; + + if (i == 9) + fail_unless(pa.mark_end_of_subjob(&pjob) == true); + else + fail_unless(pa.mark_end_of_subjob(&pjob) == false); + } + + // Make sure a repeated index doesn't crash + pjob.ji_wattr[JOB_ATR_job_array_id].at_val.at_long = 1; + fail_unless(pa.mark_end_of_subjob(&pjob) == false); + } +END_TEST + + START_TEST(test_initialize_uncreated_ids) { job_array pa; @@ -203,6 +239,7 @@ Suite *job_array_suite(void) tcase_add_test(tc_core, update_array_values_test); tcase_add_test(tc_core, test_set_slot_limit); tcase_add_test(tc_core, test_initialize_uncreated_ids); + tcase_add_test(tc_core, test_mark_end_of_subjob); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/job_func/scaffolding.c b/src/test/job_func/scaffolding.c index f4637d67d0..886832c64e 100644 --- a/src/test/job_func/scaffolding.c +++ b/src/test/job_func/scaffolding.c @@ -715,5 +715,10 @@ bool job_array::is_deleted() const return(this->being_deleted); } +bool job_array::mark_end_of_subjob(job *pjob) + { + return(false); + } + batch_request::~batch_request() {} diff --git a/src/test/job_recov/scaffolding.c b/src/test/job_recov/scaffolding.c index e2f57d65b8..13f8ff80fc 100644 --- a/src/test/job_recov/scaffolding.c +++ b/src/test/job_recov/scaffolding.c @@ -534,6 +534,11 @@ bool job_array::is_deleted() const return(this->being_deleted); } +bool job_array::mark_end_of_subjob(job *pjob) + { + return(false); + } + int check_default_gpu_mode_str(pbs_attribute *pattr, void *pobj, int mode) { return(PBSE_NONE); From 82f8c244f0d07e1f6cee61032a3a23de8b2c378c Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 13 Dec 2016 15:54:11 -0700 Subject: [PATCH 104/312] TRQ-3609. Make qsub fail if the working directory isn't valid. --- src/cmds/qsub_functions.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 5f06ccb9cd..dda47231d7 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -794,7 +794,6 @@ void validate_pbs_o_workdir( { job_data *tmp_job_info = NULL; const char *the_val = NULL; - char null_val[] = "\0"; char tmp_dir[MAXPATHLEN] = {""}; if (hash_find(job_attr, ATTR_init_work_dir, &tmp_job_info) == FALSE) @@ -807,11 +806,25 @@ void validate_pbs_o_workdir( if ((the_dir = getcwd(tmp_dir, MAXPATHLEN)) != NULL) the_val = the_dir; else - the_val = null_val; + { + // Current working directory is deleted. Fail + fprintf(stderr, "qsub: Cannot obtain the current directory.\nPlease submit from a valid directory.\n"); + exit(3); + } } } else + { + struct stat sb; + the_val = tmp_job_info->value.c_str(); + if ((stat(the_val, &sb) != 0) || + (!(S_ISDIR(sb.st_mode)))) + { + fprintf(stderr, "qsub: Requested working directory '%s' is not a valid directory\nPlease specify a valid working directory.\n", the_val); + exit(3); + } + } hash_add_or_exit(job_attr, ATTR_pbs_o_workdir, the_val, ENV_DATA); hash_add_or_exit(job_attr, ATTR_init_work_dir, the_val, ENV_DATA); From 44718f101e95de0a99c8e4f871a213def6a47539 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 14 Dec 2016 11:51:39 -0700 Subject: [PATCH 105/312] TRQ-1996. Remove unnecessary libraries from pbs_demux. --- src/resmom/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index 70d7492bb0..06943b304b 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -63,7 +63,7 @@ endif pbs_demux_SOURCES = pbs_demux.c # use LDADD here so that we don't have to link MOM with libcr -pbs_demux_LDADD = $(BLCR_LDFLAGS) @PBS_MACH@/libmommach.a $(MOMLIBS) $(PBS_LIBS) +pbs_demux_LDADD = $(BLCR_LDFLAGS) @PBS_MACH@/libmommach.a $(PBS_LIBS) pbs_demux_CPPFLAGS = $(BLCR_CPPFLAGS) $(AM_CPPFLAGS) install-exec-hook: From 3943c600f2402d4e3264cf0963da25dd0f845e45 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 14 Dec 2016 14:27:53 -0700 Subject: [PATCH 106/312] TRQ-3886. Fix the regression from the other check-in. Arrays weren't being purged. --- src/server/job_func.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server/job_func.c b/src/server/job_func.c index e7eed7d92a..c8950a3650 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -1939,6 +1939,10 @@ int svr_job_purge( { if ((do_delete_array = pa->mark_end_of_subjob(pjob)) == false) array_save(pa); + else + { + snprintf(array_id, sizeof(array_id), "%s", pa->ai_qs.parent_id); + } unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); } From 1643f78a12b25bc98c1cba436654904946190aa4 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 14 Dec 2016 16:24:03 -0700 Subject: [PATCH 107/312] TRQ-3886. Fix another regression - validate the array index from array size, not the number of jobs. --- src/server/job_array.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/job_array.cpp b/src/server/job_array.cpp index 63f06c9db0..1721d0a872 100644 --- a/src/server/job_array.cpp +++ b/src/server/job_array.cpp @@ -593,7 +593,7 @@ bool job_array::mark_end_of_subjob( index = pjob->ji_wattr[JOB_ATR_job_array_id].at_val.at_long; if ((index >= 0) && - (index < this->ai_qs.num_jobs)) + (index < this->ai_qs.array_size)) { if (this->job_ids[index] != NULL) { From 8d89d7fbbd26e808b2c9023716bdb0ab7a1cf823 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 14 Dec 2016 16:33:58 -0700 Subject: [PATCH 108/312] TRQ-3332 - add array slot limit to qstat output --- src/include/array.h | 2 +- src/lib/Libattr/attr_fn_hold.c | 14 ++++++++++++++ src/server/job_array.cpp | 20 ++++++++++++++++---- src/test/attr_fn_hold/test_uut.c | 31 +++++++++++++++++++++++-------- src/test/job_array/test_uut.c | 6 +++++- 5 files changed, 59 insertions(+), 14 deletions(-) diff --git a/src/include/array.h b/src/include/array.h index 89b8676986..74050d0a25 100644 --- a/src/include/array.h +++ b/src/include/array.h @@ -160,7 +160,7 @@ class job_array ~job_array(); int parse_array_request(const char *request); - int set_slot_limit(char *request); + int set_slot_limit(const char *request); int set_idle_slot_limit(long requested_limit); void set_submit_host(const char *submit_host); void set_owner(const char *owner); diff --git a/src/lib/Libattr/attr_fn_hold.c b/src/lib/Libattr/attr_fn_hold.c index 3fed470150..16ad88d406 100644 --- a/src/lib/Libattr/attr_fn_hold.c +++ b/src/lib/Libattr/attr_fn_hold.c @@ -144,6 +144,14 @@ int decode_hold( patr->at_val.at_long |= HOLD_s; break; + case 'a': + patr->at_val.at_long |= HOLD_a; + break; + + case 'l': + patr->at_val.at_long |= HOLD_l; + break; + default: return (PBSE_BADATVAL); } @@ -208,6 +216,12 @@ int encode_hold( if (attr->at_val.at_long & HOLD_u) *(pal->al_value + i++) = 'u'; + + if (attr->at_val.at_long & HOLD_a) + *(pal->al_value + i++) = 'a'; + + if (attr->at_val.at_long & HOLD_l) + *(pal->al_value + i++) = 'l'; } pal->al_flags = attr->at_flags; diff --git a/src/server/job_array.cpp b/src/server/job_array.cpp index 1721d0a872..575c13118b 100644 --- a/src/server/job_array.cpp +++ b/src/server/job_array.cpp @@ -130,18 +130,22 @@ void job_array::set_submit_host( int job_array::set_slot_limit( - char *request) + const char *request) { char *pcnt; + char *request_copy; long max_limit; char log_buf[LOCAL_LOG_BUF_SIZE]; + if ((request_copy = strdup(request)) == NULL) + return(-1); + /* check for a max slot limit */ if (get_svr_attr_l(SRV_ATR_MaxSlotLimit, &max_limit) != PBSE_NONE) max_limit = NO_SLOT_LIMIT; - if ((pcnt = strchr(request,'%')) != NULL) + if ((pcnt = strchr(request_copy, '%')) != NULL) { /* remove '%' from the request, or else it can't be parsed */ while (*pcnt == '%') @@ -164,6 +168,7 @@ int job_array::set_slot_limit( max_limit); log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_JOB, this->ai_qs.parent_id, log_buf); + free(request_copy); return(INVALID_SLOT_LIMIT); } } @@ -177,6 +182,7 @@ int job_array::set_slot_limit( this->ai_qs.slot_limit = max_limit; } + free(request_copy); return(PBSE_NONE); } /* END set_slot_limit() */ @@ -239,11 +245,17 @@ int job_array::parse_array_request( long max_array_size; char log_buf[LOCAL_LOG_BUF_SIZE]; this->uncreated_ids.clear(); + std::string request_copy(request); + int loc; + + // remove slot limit (begins with %) if present + if ((loc = request_copy.find_first_of("%")) >= 0) + request_copy.erase(loc); - if ((rc = translate_range_string_to_vector(request, this->uncreated_ids)) != PBSE_NONE) + if ((rc = translate_range_string_to_vector(request_copy.c_str(), this->uncreated_ids)) != PBSE_NONE) return(rc); - this->ai_qs.range_str = request; + this->ai_qs.range_str = request_copy.c_str(); this->ai_qs.num_jobs = this->uncreated_ids.size(); // size of array is the biggest index + 1 diff --git a/src/test/attr_fn_hold/test_uut.c b/src/test/attr_fn_hold/test_uut.c index f8f6681886..c432081b73 100644 --- a/src/test/attr_fn_hold/test_uut.c +++ b/src/test/attr_fn_hold/test_uut.c @@ -39,6 +39,13 @@ START_TEST(test_one) memset(&attr,0,sizeof(attr)); fail_unless(decode_hold(&attr,NULL,NULL,"banana",0) == PBSE_BADATVAL); + memset(&attr,0,sizeof(attr)); + fail_unless(decode_hold(&attr,NULL,NULL,"a",0) == 0); + fail_unless(attr.at_flags == (ATR_VFLAG_SET|ATR_VFLAG_MODIFY)); + + memset(&attr,0,sizeof(attr)); + fail_unless(decode_hold(&attr,NULL,NULL,"l",0) == 0); + fail_unless(attr.at_flags == (ATR_VFLAG_SET|ATR_VFLAG_MODIFY)); } END_TEST @@ -47,14 +54,22 @@ START_TEST(test_two) svrattrl *attrl = attrlist_create("Fred","Wilma",20); pbs_attribute f; pbs_attribute t; - - memset(&f,0,sizeof(f)); - decode_hold(&f,NULL,NULL,"so",0); - memset(&t,0,sizeof(t)); - decode_hold(&t,NULL,NULL,"nu",0); - - fail_unless(encode_hold(&f,(tlist_head *)attrl,"FALSE",NULL,0,0) == 1); - fail_unless(encode_hold(&t,(tlist_head *)attrl,"TRUE",NULL,0,0) == 1); + pbs_attribute t2; + pbs_attribute t3; + + memset(&f, 0, sizeof(f)); + decode_hold(&f, NULL, NULL, "so", 0); + memset(&t, 0, sizeof(t)); + decode_hold(&t, NULL, NULL, "nu", 0); + memset(&t2, 0, sizeof(t2)); + decode_hold(&t2, NULL, NULL, "a", 0); + memset(&t3, 0, sizeof(t3)); + decode_hold(&t3, NULL, NULL, "l", 0); + + fail_unless(encode_hold(&f, (tlist_head *)attrl, "FALSE", NULL, 0, 0) == 1); + fail_unless(encode_hold(&t, (tlist_head *)attrl, "TRUE", NULL, 0, 0) == 1); + fail_unless(encode_hold(&t2, (tlist_head *)attrl, "TRUE", NULL, 0, 0) == 1); + fail_unless(encode_hold(&t3, (tlist_head *)attrl, "TRUE", NULL, 0, 0) == 1); } END_TEST diff --git a/src/test/job_array/test_uut.c b/src/test/job_array/test_uut.c index 8c062a2894..9d9af011df 100644 --- a/src/test/job_array/test_uut.c +++ b/src/test/job_array/test_uut.c @@ -1,5 +1,6 @@ #include "license_pbs.h" /* See here for the software license */ #include +#include #include #include "pbs_job.h" @@ -72,6 +73,9 @@ START_TEST(test_set_slot_limit) request = strdup("0-99%2"); fail_unless(pa.set_slot_limit(request) == PBSE_NONE); fail_unless(pa.ai_qs.slot_limit == 2); + + // make sure request not modified + fail_unless(strcmp(request, "0-99%2") == 0); max_slot = 1; request = strdup("0-99%2"); @@ -113,7 +117,7 @@ START_TEST(test_parse_array_request) // submit a valid array array_size = 10000; - fail_unless(pa.parse_array_request("0-9") == PBSE_NONE); + fail_unless(pa.parse_array_request("0-9%1") == PBSE_NONE); fail_unless(pa.ai_qs.array_size == 10); fail_unless(pa.ai_qs.num_jobs == 10); fail_unless(pa.ai_qs.range_str == "0-9"); From 33288f1d54863bf39a7149f9db42129e6ff8bc87 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 14 Dec 2016 18:05:10 -0700 Subject: [PATCH 109/312] Change torque-build to run under bash --- buildutils/torque-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index 6b59cd8886..cf24018908 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash POSIXLY_CORRECT=1 From 5b1d7922806da513bc1290ab01559dc7cd4e2260 Mon Sep 17 00:00:00 2001 From: Sarah Sather Date: Wed, 14 Dec 2016 20:20:16 -0700 Subject: [PATCH 110/312] DOC-2819 pbs_server_attributes document scheduler_iteration has no effect --- doc/man7/pbs_server_attributes.7.in | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/doc/man7/pbs_server_attributes.7.in b/doc/man7/pbs_server_attributes.7.in index 22215f2f81..d8cac0dd0a 100644 --- a/doc/man7/pbs_server_attributes.7.in +++ b/doc/man7/pbs_server_attributes.7.in @@ -789,14 +789,7 @@ ignored by pbs_server. [internal type: string] .Ig .Al scheduler_iteration -The time, in seconds, between iterations of attempts by the batch server -to schedule jobs. On each iteration, the server examines the available -resources and runnable jobs to see if a job can be initiated. This -examination also occurs whenever a running batch job terminates or a -new job is placed in the queued state in an execution queue. -Format: integer seconds; default value: 10 minutes, set by -.Sc PBS_SCHEDULE_CYCLE -in server_limits.h. +Has no effect. .if !\n(Pb .ig Ig [internal type: integer, displays as name defined below] .Ig From ce3b78fafc4c4ee55121e939c76f732aebd5dfb1 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 15 Dec 2016 15:30:45 -0700 Subject: [PATCH 111/312] TRQ-3711. Fix false positives around job start failures for setting nodes down. Conflicts: src/server/req_jobobit.c --- src/server/req_jobobit.c | 4 ++-- src/server/req_runjob.c | 43 +++++++++++++++++++++++++++++++++++++++- src/server/svr_movejob.c | 24 +++++++++++++++++++--- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/server/req_jobobit.c b/src/server/req_jobobit.c index 9146215a0f..72531ae8ee 100644 --- a/src/server/req_jobobit.c +++ b/src/server/req_jobobit.c @@ -1559,11 +1559,11 @@ int handle_exited( } delete preq; + + preq = NULL; } else job_mutex.unlock(); - - preq = NULL; if ((pjob = svr_find_job(job_id, TRUE)) == NULL) return(PBSE_JOBNOTFOUND); diff --git a/src/server/req_runjob.c b/src/server/req_runjob.c index 17d2d4188d..81c69cbae0 100644 --- a/src/server/req_runjob.c +++ b/src/server/req_runjob.c @@ -1369,6 +1369,43 @@ int svr_strtjob2( +/* + * get_message_from_mom_err() + * + * @param mom_err (I) - the error code sent back from the mom + * @param msg (O) - the message string to populate + */ + +void get_message_from_mom_err( + + int mom_err, + std::string &msg) + + { + switch (mom_err) + { + case JOB_EXEC_CMDFAIL: + + msg = "Failed to exec the job, please retry."; + break; + + case JOB_EXEC_STDOUTFAIL: + + msg = "Failed to open the output and error files for the job."; + break; + + case JOB_EXEC_RETRY_CGROUP: + + msg = "Failed to create the cgroups for this job."; + break; + + default: + + msg = "Mom rejected the job."; + break; + } + } // get_message_from_mom_err() + void finish_sendmom( @@ -1481,7 +1518,11 @@ void finish_sendmom( if (preq != NULL) { if (mom_err != PBSE_NONE) - req_reject(mom_err, 0, preq, node_name, "connection to mom timed out"); + { + std::string mom_err_msg; + get_message_from_mom_err(mom_err, mom_err_msg); + req_reject(mom_err, 0, preq, node_name, mom_err_msg.c_str()); + } else req_reject(PBSE_MOMREJECT, 0, preq, node_name, "connection to mom timed out"); } diff --git a/src/server/svr_movejob.c b/src/server/svr_movejob.c index 447d17e793..75456dcd8f 100644 --- a/src/server/svr_movejob.c +++ b/src/server/svr_movejob.c @@ -939,6 +939,11 @@ void log_commit_error( mom_err, pbse_to_txt(mom_err), (err_text != NULL) ? err_text : "N/A"); errno2 = mom_err; } + else if (mom_err < 0) + { + // Mom failed with a JOB_EXEC_* code + errno2 = mom_err; + } else { sprintf(log_buf, "send_job commit failed, rc=%d (%s)", @@ -1117,6 +1122,11 @@ int send_job_over_network( +/* + * send_job_over_network_with_retries() + * + */ + int send_job_over_network_with_retries( char *job_id, @@ -1382,7 +1392,7 @@ int send_job_work( my_err, &mom_err); - if (Timeout == TRUE) + if (Timeout == true) { /* 10 indicates that job migrate timed out, server will mark node down * and abort the job - see post_sendmom() */ @@ -1404,10 +1414,18 @@ int send_job_work( if (type == MOVE_TYPE_Exec) { + // If the job failed to submit, we only want to call it a network error if we didn't + // get a reply from the mom, in which case mom_err will still be set to PBSE_NONE. Otherwise, + // it was not a network error and the node is responding. + int fail_count_rc = PBSE_NONE; + if ((rc != PBSE_NONE) && + (mom_err == PBSE_NONE)) + fail_count_rc = mom_err; + if (node_name != NULL) - update_failure_counts(node_name, rc); + update_failure_counts(node_name, fail_count_rc); else - update_failure_counts(job_destin, rc); + update_failure_counts(job_destin, fail_count_rc); } send_job_work_end: From 27ce22901e8d067609d827bb9dbc547cba6549a6 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 15 Dec 2016 22:34:05 +0000 Subject: [PATCH 112/312] TRQ-3888 - add -G option to torque-build --- buildutils/torque-build | 9 ++++++++- buildutils/torque.adaptive.spec.el6 | 7 +++++-- buildutils/torque.adaptive.spec.el7 | 7 +++++-- buildutils/torque.adaptive.spec.sles12 | 7 +++++-- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/buildutils/torque-build b/buildutils/torque-build index cf24018908..ed6782d48e 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -29,7 +29,8 @@ usage() { echo " /opt/coverity/default/bin/cov-build)" echo " -C|--no-coverity Do not run coverity (default)" echo "" - echo " -g|--enable-cgroups Enable cgroups" + echo " -g|--enable-cgroups Enable cgroups (default)" + echo " -G|--disable-cgroups Do not enable cgroups" echo " -u|--unit-tests Run unit-tests (default)" echo " -U|--no-unit-tests Do not run unit tests" echo "" @@ -134,6 +135,11 @@ do cgroups=1 shift ;; + -G|--disable-cgroups) + cgroups=0 + with_option="$with_option --without cgroups --without hwloc" + shift + ;; *) echo "Invalid argument '${1}' specified." >&2 usage @@ -225,6 +231,7 @@ then cgroupsopts="--enable-cgroups" configure_options="$configure_options --enable-cgroups --with-hwloc-path=$hwloc_path" else + hwloc_path=none cgroupsopts="--disable-cgroups" fi diff --git a/buildutils/torque.adaptive.spec.el6 b/buildutils/torque.adaptive.spec.el6 index 7fc84ab6c9..cd216f4bb2 100644 --- a/buildutils/torque.adaptive.spec.el6 +++ b/buildutils/torque.adaptive.spec.el6 @@ -63,9 +63,12 @@ unset CONFFLAGS \ CONFFLAGS_FILE=`mktemp %{_tmppath}/confflags.XXXX` \ ./configure --help 2>&1 > ${CONFFLAGS_FILE} \ - if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + if test %{_hwlocpath} != none \ then \ - CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + fi \ fi \ if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ then \ diff --git a/buildutils/torque.adaptive.spec.el7 b/buildutils/torque.adaptive.spec.el7 index 74dd1b2065..e8793628d8 100644 --- a/buildutils/torque.adaptive.spec.el7 +++ b/buildutils/torque.adaptive.spec.el7 @@ -63,9 +63,12 @@ unset CONFFLAGS \ CONFFLAGS_FILE=`mktemp %{_tmppath}/confflags.XXXX` \ ./configure --help 2>&1 > ${CONFFLAGS_FILE} \ - if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + if test %{_hwlocpath} != none \ then \ - CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + fi \ fi \ if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ then \ diff --git a/buildutils/torque.adaptive.spec.sles12 b/buildutils/torque.adaptive.spec.sles12 index 74dd1b2065..18ebaeecd6 100644 --- a/buildutils/torque.adaptive.spec.sles12 +++ b/buildutils/torque.adaptive.spec.sles12 @@ -63,9 +63,12 @@ unset CONFFLAGS \ CONFFLAGS_FILE=`mktemp %{_tmppath}/confflags.XXXX` \ ./configure --help 2>&1 > ${CONFFLAGS_FILE} \ - if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + if test %{_hwlocpath} != none \ then \ - CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + fi \ fi \ if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ then \ From 3d4d3cd4a0b970121ad93475b32c2439388c7729 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 15 Dec 2016 16:46:17 -0700 Subject: [PATCH 113/312] TRQ-3729. Add the -L submission, if present, to the accounting log. Conflicts: src/include/pbs_job.h src/server/job_attr_def.c --- src/cmds/qsub_functions.c | 8 ++++++++ src/include/pbs_ifl.h | 1 + src/include/pbs_job.h | 1 + src/server/accounting.c | 8 ++++++++ src/server/job_attr_def.c | 13 +++++++++++++ 5 files changed, 31 insertions(+) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index dda47231d7..365cb9fc65 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -93,6 +93,7 @@ const char *checkpoint_strings = "n,c,s,u,none,shutdown,periodic,enabled,interva char *alternate_dependency; int alternate_data_type; complete_req cr; +std::string L_request; /* adapted from openssh */ /* The parameter was EMsg, but was never used. @@ -1121,6 +1122,7 @@ void add_new_request_if_present( std::string req_str; cr.toString(req_str); hash_add_or_exit(ji->job_attr, ATTR_req_information, req_str.c_str(), CMDLINE_DATA); + hash_add_or_exit(ji->job_attr, ATTR_L_request, L_request.c_str(), CMDLINE_DATA); } } // END add_new_request_if_present() @@ -2576,6 +2578,12 @@ void process_opt_L( { char err_buf[MAXLINE*2]; + if (L_request.size() > 0) + L_request += " "; + + L_request += "-L "; + L_request += cmd_arg; + if (strncmp(cmd_arg, "tasks=", 6)) { snprintf(err_buf, sizeof(err_buf), "qsub: illegal -L value: '%s'", cmd_arg); diff --git a/src/include/pbs_ifl.h b/src/include/pbs_ifl.h index f10eed1e18..1f4bbf5d2a 100644 --- a/src/include/pbs_ifl.h +++ b/src/include/pbs_ifl.h @@ -380,6 +380,7 @@ #define ATTR_cpustr "cpuset_string" #define ATTR_memstr "memset_string" +#define ATTR_L_request "L_Request" #define ATTR_user_kill_delay "user_kill_delay" #define ATTR_idle_slot_limit "idle_slot_limit" #define ATTR_default_gpu_mode "default_gpu_mode" diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 7452abe56f..32d255ee38 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -436,6 +436,7 @@ enum job_atr JOB_ATR_memset_string, JOB_ATR_user_kill_delay, JOB_ATR_idle_slot_limit, + JOB_ATR_LRequest, JOB_ATR_UNKN, /* the special "unknown" type */ JOB_ATR_LAST /* This MUST be LAST */ }; diff --git a/src/server/accounting.c b/src/server/accounting.c index ed9600d9f9..d084491cde 100644 --- a/src/server/accounting.c +++ b/src/server/accounting.c @@ -274,6 +274,14 @@ int acct_job( free(pal); } /* END while (pal != NULL) */ + if ((pjob->ji_wattr[JOB_ATR_LRequest].at_val.at_str != NULL) && + ((pjob->ji_wattr[JOB_ATR_LRequest].at_flags & ATR_VFLAG_SET) != 0)) + { + ds += "Resource_Request_2.0="; + ds += pjob->ji_wattr[JOB_ATR_LRequest].at_val.at_str; + ds += " "; + } + #ifdef ATTR_X_ACCT /* x attributes */ diff --git a/src/server/job_attr_def.c b/src/server/job_attr_def.c index 5f1b6ada6c..c46e0b26cc 100644 --- a/src/server/job_attr_def.c +++ b/src/server/job_attr_def.c @@ -1318,6 +1318,19 @@ attribute_def job_attr_def[] = READ_WRITE, ATR_TYPE_LONG, PARENT_TYPE_JOB}, + + // JOB_ATR_LRequest + {(char *)ATTR_L_request, // "L_Request" + decode_str, + encode_str, + set_str, + comp_str, + free_str, + NULL_FUNC, + READ_WRITE, + ATR_TYPE_STR, + PARENT_TYPE_JOB, + }, /* JOB_ATR_UNKN - THIS MUST BE THE LAST ENTRY */ { (char *)"_other_", From 0de4c3784bede52466d2fed86a4513febb31769a Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 3 Jan 2017 17:50:21 -0700 Subject: [PATCH 114/312] TRQ-3886. Tweak job array cleanup. --- src/server/job_array.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/server/job_array.cpp b/src/server/job_array.cpp index 575c13118b..f5261a7888 100644 --- a/src/server/job_array.cpp +++ b/src/server/job_array.cpp @@ -611,14 +611,14 @@ bool job_array::mark_end_of_subjob( { free(this->job_ids[index]); this->job_ids[index] = NULL; - - this->ai_qs.num_purged++; - - if ((this->ai_qs.num_purged == this->ai_qs.num_jobs) || - ((this->is_deleted() == true) && - (this->ai_qs.num_idle == 0))) - no_more_subjobs = true; } + + this->ai_qs.num_purged++; + + if ((this->ai_qs.num_purged == this->ai_qs.num_jobs) || + ((this->is_deleted() == true) && + (this->ai_qs.num_idle == 0))) + no_more_subjobs = true; } } } From a2bcb03e877e48fbfa1d3e26e9ecb9df568e2a9d Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 4 Jan 2017 13:21:45 -0700 Subject: [PATCH 115/312] TRQ-3830. Make conflicting defaults not get applied. For example, if a queue has a proc default and a job submits a nodes request, don't apply the proc default as this will invalidate the job. --- src/cmds/qsub_functions.c | 5 +++ src/server/svr_jobfunc.c | 65 +++++++++++++++++++++++++++++---- src/test/svr_jobfunc/test_uut.c | 57 +++++++++++++++++++++++++++-- 3 files changed, 116 insertions(+), 11 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 365cb9fc65..f5a1117acb 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -898,6 +898,11 @@ void validate_qsub_host_pbs_o_server( } /* END validate_qsub_host_pbs_o_server() */ + +/* + * are_mpp_present() + */ + int are_mpp_present( job_data_container *resources, diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index 644d1f3e37..a39cbb58cd 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -3137,7 +3137,6 @@ int job_set_wait( - /* * default_std - make the default name for standard output or error * "job_name".[e|o]job_sequence_number @@ -3191,8 +3190,6 @@ static void default_std( - - /* * prefix_std_file - build the fully prefixed default file name for the job's * standard output or error: @@ -3271,7 +3268,6 @@ const char *prefix_std_file( - /* * add_std_filename - add the default file name for the job's * standard output or error: @@ -3344,17 +3340,66 @@ void get_jobowner( *(to + i) = '\0'; return; - } + } // END get_jobowner() + + + +/* + * is_execution_slot_resource() + * + * @param r - the resource we're checking + * @return true if this resource is an execution slot related request + */ + +inline bool is_execution_slot_resource( + + const resource &r) + + { + return((!strcmp(r.rs_defin->rs_name, "nodes")) || + (!strcmp(r.rs_defin->rs_name, "procs")) || + (!strcmp(r.rs_defin->rs_name, "size")) || + (!strcmp(r.rs_defin->rs_name, "ncpus"))); + } // is_execution_slot_resource() + + +/* + * contains_execution_slot_request() + * + * @param jb - the resources requested by the job + */ +bool contains_execution_slot_request( + + pbs_attribute *jb) + + { + bool execution_slot_request = false; + + std::vector *job_resc = (std::vector *)jb->at_val.at_ptr; + + for (size_t i = 0; i < job_resc->size(); i++) + { + resource &r = job_resc->at(i); + + if (is_execution_slot_resource(r)) + { + execution_slot_request = true; + break; + } + } + + return(execution_slot_request); + } // END contains_execution_slot_request() /** * @see set_resc_deflt() - parent * - * @param jb - * @param *dflt + * @param jb - the job's resources + * @param dflt - the specified default resources */ void set_deflt_resc( @@ -3376,10 +3421,16 @@ void set_deflt_resc( if (resc_deflt != NULL) { + bool es_spec = contains_execution_slot_request(jb); + for (size_t i = 0; i < resc_deflt->size(); i++) { resource &r = resc_deflt->at(i); + if ((es_spec == true) && + (is_execution_slot_resource(r))) + continue; + prescjb = find_resc_entry(jb, r.rs_defin); if ((prescjb == NULL) || diff --git a/src/test/svr_jobfunc/test_uut.c b/src/test/svr_jobfunc/test_uut.c index 67b45392c6..50ffb0b740 100644 --- a/src/test/svr_jobfunc/test_uut.c +++ b/src/test/svr_jobfunc/test_uut.c @@ -23,6 +23,7 @@ int chk_mppnodect(resource *mppnodect, pbs_queue *pque, long nppn, long mpp_widt void job_wait_over(struct work_task *); bool is_valid_state_transition(job &pjob, int newstate, int newsubstate); bool has_conflicting_resource_requests(job *pjob, pbs_queue *pque); +bool contains_execution_slot_request(pbs_attribute *jb); extern int decrement_count; extern job napali_job; @@ -33,7 +34,7 @@ extern bool possible; extern bool get_jobs_queue_force_null; extern std::string global_log_buf; -void add_resc_attribute(pbs_attribute *pattr, resource_def *prdef, const char *value) +void add_resc_attribute(pbs_attribute *pattr, resource_def *prdef, const char *value, long v) { if (pattr->at_val.at_ptr == NULL) { @@ -44,13 +45,59 @@ void add_resc_attribute(pbs_attribute *pattr, resource_def *prdef, const char *v resource rsc; rsc.rs_defin = prdef; rsc.rs_value.at_type = prdef->rs_type; - rsc.rs_value.at_val.at_str = strdup(value); + + if (value != NULL) + rsc.rs_value.at_val.at_str = strdup(value); + else + rsc.rs_value.at_val.at_long = v; + resources->push_back(rsc); pattr->at_flags = ATR_VFLAG_SET; pattr->at_val.at_ptr = resources; } +START_TEST(contains_execution_slot_request_test) + { + pbs_attribute pattr; + + resource_def rdef[5]; + rdef[0].rs_name = strdup("nodes"); + rdef[1].rs_name = strdup("walltime"); + rdef[2].rs_name = strdup("ncpus"); + rdef[3].rs_name = strdup("procs"); + rdef[4].rs_name = strdup("size"); + memset(&pattr, 0, sizeof(pattr)); + + // Add walltime to the job request + add_resc_attribute(&pattr, rdef + 1, NULL, 60); + fail_unless(contains_execution_slot_request(&pattr) == false); + + // Now add nodes + add_resc_attribute(&pattr, rdef, "2:ppn=10", 60); + fail_unless(contains_execution_slot_request(&pattr) == true); + + // Make a new request with ncpus + pbs_attribute ncpus_attr; + memset(&ncpus_attr, 0, sizeof(ncpus_attr)); + add_resc_attribute(&ncpus_attr, rdef + 2, NULL, 6); + fail_unless(contains_execution_slot_request(&ncpus_attr) == true); + + // Make a new request with procs + pbs_attribute procs_attr; + memset(&procs_attr, 0, sizeof(procs_attr)); + add_resc_attribute(&procs_attr, rdef + 3, NULL, 4); + fail_unless(contains_execution_slot_request(&procs_attr) == true); + + // Finally, make one with size + pbs_attribute size_attr; + memset(&size_attr, 0, sizeof(size_attr)); + add_resc_attribute(&size_attr, rdef + 4, NULL, 64); + fail_unless(contains_execution_slot_request(&size_attr) == true); + } +END_TEST + + START_TEST(test_numa_task_exceeds_resources) { job *pjob = (job *)calloc(1, sizeof(job)); @@ -429,14 +476,15 @@ START_TEST(chk_resc_min_limits_test) memset(&test_attribute, 0, sizeof(test_attribute)); memset(&test_queue, 0, sizeof(test_queue)); - add_resc_attribute(&test_queue.qu_attr[QA_ATR_ResourceMin], &nodes_resource_def, "3"); - add_resc_attribute(&test_attribute, &nodes_resource_def, "2:ppn=1+1:ppn=4"); + add_resc_attribute(&test_queue.qu_attr[QA_ATR_ResourceMin], &nodes_resource_def, "3", 1); + add_resc_attribute(&test_attribute, &nodes_resource_def, "2:ppn=1+1:ppn=4", 1); result = chk_resc_limits(&test_attribute, &test_queue, message); fail_unless(result == PBSE_NONE, "Fail to approve queue minimum resource"); } END_TEST + START_TEST(svr_chkque_test) { struct job test_job; @@ -733,6 +781,7 @@ Suite *svr_jobfunc_suite(void) tc_core = tcase_create("set_chkpt_deflt_test"); tcase_add_test(tc_core, set_chkpt_deflt_test); + tcase_add_test(tc_core, contains_execution_slot_request_test); suite_add_tcase(s, tc_core); tc_core = tcase_create("set_statechar_test"); From f25b8d37643513f86f44c8a819f03ca9616e41d7 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 4 Jan 2017 14:51:24 -0700 Subject: [PATCH 116/312] TRQ-2212. Add a --logfile_dir= option for trqauthd. --- src/daemon_client/trq_auth_daemon.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/daemon_client/trq_auth_daemon.c b/src/daemon_client/trq_auth_daemon.c index 248a09c7ad..57a8a17d7d 100644 --- a/src/daemon_client/trq_auth_daemon.c +++ b/src/daemon_client/trq_auth_daemon.c @@ -44,6 +44,7 @@ bool use_log = true; bool daemonize_server = true; static int changed_msg_daem = 0; static char *active_pbs_server; +std::string log_file_path = ""; /* Get the name of the active pbs_server */ int load_trqauthd_config( @@ -129,7 +130,14 @@ int init_trqauth_log(const char *server_port) log_get_set_eventclass(&eventclass, SETV); initialize_globals_for_log(server_port); - sprintf(path_log, "%s/%s", path_home, TRQ_LOGFILES); + if(log_file_path == "") + { + sprintf(path_log, "%s/%s", path_home, TRQ_LOGFILES); + } + else + { + sprintf(path_log, "%s", log_file_path.c_str()); + } if ((mkdir(path_log, 0755) == -1) && (errno != EEXIST)) { openlog("daemonize_trqauthd", LOG_PID | LOG_NOWAIT, LOG_DAEMON); @@ -264,6 +272,7 @@ void parse_command_line(int argc, char **argv) {"about", no_argument, 0, 0 }, {"help", no_argument, 0, 0 }, {"version", no_argument, 0, 0 }, + {"logfile_dir", required_argument,0, 0 }, {0, 0, 0, 0 } }; @@ -295,6 +304,9 @@ void parse_command_line(int argc, char **argv) fprintf(stderr, "Version: %s \nCommit: %s\n", VERSION, GIT_HASH); exit(0); break; + case 3: /* logfile */ + log_file_path = optarg; + break; } break; From 835da66b171112e2af5dd3a7f49dd6c9fe6de1ac Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 5 Jan 2017 17:11:27 -0700 Subject: [PATCH 117/312] TRQ-3213. Set the gpu mode for -l procs jobs as well. --- src/server/node_manager.c | 4 +- src/server/req_runjob.c | 107 ++++++++++++++++++++++---------------- 2 files changed, 65 insertions(+), 46 deletions(-) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 138b78354e..e8bd41010c 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -2100,7 +2100,7 @@ int proplist( char name_storage[80]; char *pname; char *pequal; - int have_gpus = FALSE; + bool have_gpus = false; char log_buf[LOCAL_LOG_BUF_SIZE]; *node_req = 1; /* default to 1 processor per node */ @@ -2155,7 +2155,7 @@ int proplist( return(1); } - have_gpus = TRUE; + have_gpus = true; gpu_err_reset = FALSE; /* default to no */ // default value if no other gets specified diff --git a/src/server/req_runjob.c b/src/server/req_runjob.c index 81c69cbae0..8b128b8dba 100644 --- a/src/server/req_runjob.c +++ b/src/server/req_runjob.c @@ -1913,6 +1913,35 @@ int set_job_exec_info( +void set_mode_from_string( + + const char *mode_string, + std::string &mode) + + { + if (strstr(mode_string, "exclusive_thread")) + { + mode = ":exclusive_thread"; + } + else if (strstr(mode_string, "exclusive_process")) + { + mode = ":exclusive_process"; + } + else if (strstr(mode_string, "exclusive")) + { + mode = ":exclusive"; + } + else if (strstr(mode_string, "default")) + { + mode = ":default"; + } + else if (strstr(mode_string, "shared")) + { + mode = ":shared"; + } + } // END set_mode_from_string() + + char *get_correct_spec_string( @@ -1920,18 +1949,18 @@ char *get_correct_spec_string( job *pjob) { - char mode[20]; - char *mode_string; - char *request; - char *correct_spec = NULL; - char *outer_plus; - char *plus; - char *one_req; - int num_gpu_reqs; - char *gpu_req; - int len; - resource *pres = NULL; - char log_buf[LOCAL_LOG_BUF_SIZE]; + std::string mode; + char *mode_string = NULL; + char *request; + char *correct_spec = NULL; + char *outer_plus = NULL; + char *plus; + char *one_req; + int num_gpu_reqs; + char *gpu_req; + int len; + resource *pres = NULL; + char log_buf[LOCAL_LOG_BUF_SIZE]; /* check to see if there is a gpus request. If so moab * sripted the mode request if it existed. We need to @@ -1942,8 +1971,15 @@ char *get_correct_spec_string( { /* Build our host list from what is in the job attrs */ pres = find_resc_entry( - &pjob->ji_wattr[(int)JOB_ATR_resource], + &pjob->ji_wattr[JOB_ATR_resource], find_resc_def(svr_resc_def, "neednodes", svr_resc_size)); + + if (pres == NULL) + { + pres = find_resc_entry( + &pjob->ji_wattr[JOB_ATR_resource], + find_resc_def(svr_resc_def, "gres", svr_resc_size)); + } if (pres != NULL) { @@ -1962,11 +1998,17 @@ char *get_correct_spec_string( if (!(gpu_req = strstr(request, ":gpus="))) { - correct_spec = strdup(given); - return(correct_spec); + if (!(gpu_req = strstr(request, "gpus:"))) + { + correct_spec = strdup(given); + return(correct_spec); + } + else + mode_string = gpu_req + 5; // strlen(gpus:) } + else + mode_string = gpu_req + 6; // strlen(:gpus=) - mode_string = gpu_req + strlen(":gpus="); while (isdigit(*mode_string)) mode_string++; @@ -1984,43 +2026,19 @@ char *get_correct_spec_string( if ((outer_plus = strchr(mode_string, '+')) != NULL) *outer_plus = '\0'; - + /* * The neednodes original value may have non gpu things in it, so we * can not rely on the requested gpu mode being the first item in the * the string after the gpus=x:. */ - - if (strstr(mode_string, "exclusive_thread")) - { - strcpy(mode, ":exclusive_thread"); - } - else if (strstr(mode_string, "exclusive_process")) - { - strcpy(mode, ":exclusive_process"); - } - else if (strstr(mode_string, "exclusive")) - { - strcpy(mode, ":exclusive"); - } - else if (strstr(mode_string, "default")) - { - strcpy(mode, ":default"); - } - else if (strstr(mode_string, "shared")) - { - strcpy(mode, ":shared"); - } - else - { - strcpy(mode, ""); - } + set_mode_from_string(mode_string, mode); if (outer_plus != NULL) *outer_plus = '+'; /* now using the actual length of requested gpu mode */ - len = strlen(given) + 1 + (num_gpu_reqs * strlen(mode)); + len = strlen(given) + 1 + (num_gpu_reqs * mode.size()); if ((correct_spec = (char *)calloc(1, len)) != NULL) { one_req = given; @@ -2034,7 +2052,7 @@ char *get_correct_spec_string( strcat(correct_spec, one_req); if (strstr(one_req, ":gpus") != NULL) - strcat(correct_spec, mode); + strcat(correct_spec, mode.c_str()); if (plus != NULL) { @@ -2045,6 +2063,7 @@ char *get_correct_spec_string( one_req = NULL; } } + if ((LOGLEVEL >= 7) && (correct_spec != NULL) && (correct_spec[0] != '\0')) { sprintf(log_buf, "%s: job gets adjusted gpu node spec of '%s'", From f82c462f361d1ac0e14260bf4f1a7f75281a5f4b Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 9 Jan 2017 11:10:23 -0700 Subject: [PATCH 118/312] DOC-3057. Fix two spelling errors in man pages. --- doc/ers/pbs_resources_all.so | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ers/pbs_resources_all.so b/doc/ers/pbs_resources_all.so index 6690f6d893..d9ea51a8b8 100644 --- a/doc/ers/pbs_resources_all.so +++ b/doc/ers/pbs_resources_all.so @@ -93,9 +93,9 @@ Each node_spec is an of nodes required of the type declared in the node_spec and a .I name or one or more -.I properity +.I property or properities desired for the nodes. -The number, the name, and each properity in the node_spec are separated by +The number, the name, and each property in the node_spec are separated by a colon ':'. If no number is specified, one (1) is assumed. Units: string. .IP From f68a6118a7f3edfd3afaece49fd3e2545ef21740 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 9 Jan 2017 11:14:34 -0700 Subject: [PATCH 119/312] TRQ-3057. Fix three more spelling errors. --- doc/ers/pbs_resources_all.so | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/ers/pbs_resources_all.so b/doc/ers/pbs_resources_all.so index d9ea51a8b8..ad8d0ffa04 100644 --- a/doc/ers/pbs_resources_all.so +++ b/doc/ers/pbs_resources_all.so @@ -94,12 +94,12 @@ of nodes required of the type declared in the node_spec and a .I name or one or more .I property -or properities desired for the nodes. +or properties desired for the nodes. The number, the name, and each property in the node_spec are separated by a colon ':'. If no number is specified, one (1) is assumed. Units: string. .IP -The name of a node is its hostname. The properities of nodes are: +The name of a node is its hostname. The properties of nodes are: .RS .IP . 2 ppn=# @@ -107,7 +107,7 @@ specifying the number of processors per node requested. Defaults to 1. .IP . 2 arbitrary string assigned by the system administrator, please check with your administrator as to the node names -and properities available to you. +and properties available to you. .RE .IP Examples: From b866eb5345fad4b1668429b51daac3544261cabd Mon Sep 17 00:00:00 2001 From: Sarah Sather Date: Mon, 9 Jan 2017 12:41:24 -0700 Subject: [PATCH 120/312] DOC-2915 pbs_mom -w now documented in manpages --- doc/man8/pbs_mom.8.in | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/man8/pbs_mom.8.in b/doc/man8/pbs_mom.8.in index c3c0e97b91..39e6d1ed38 100644 --- a/doc/man8/pbs_mom.8.in +++ b/doc/man8/pbs_mom.8.in @@ -81,8 +81,10 @@ .SH NAME pbs_mom \- start a pbs batch execution mini-server .SH SYNOPSIS -pbs_mom [\^\-a\ alarm\^] [\^\-C\ chkdirectory\^] [\^\-c\ config\^] [\^\-D\^] [\^\-d\ directory] [\^\-F\^] [\^\-h\ help] [\^\-H\ hostname] [\^\-L\ logfile] [\^\-M\ MOMport\^] [\^\-R\ RPPport\^] [\^\-p\^|\^\-q\^|\^\-r] [\^\-x] -.SH DESCRIPTION +pbs_mom [\^\-a\ alarm\^] [\^\-C\ chkdirectory\^] [\^\-c\ config\^] [\^\-D\^] [\^\-d\ directory] [\^\-F\^] [\^\-h\ help] [\^\-H\ hostname] [\^\-L\ logfile] [\^\-M\ MOMport\^] [\^\-R\ RPPport\^] [\^\-p\^|\^\-q\^|\^\-r] [\^\-w] +[\^\-x] +.LP +SH DESCRIPTION The .B pbs_mom command starts the operation of a batch \fBM\fPachine \fBO\fPriented @@ -218,6 +220,8 @@ MOM may kill a process that is not a batch session. Specifies the port number on which the pbs_server is listening for requests. If pbs_server is started with a \-p option, pbs_mom will need to use the \-S option and match the port value which was used to start pbs_server. +.IP "\-w" 16 +When started with -w, pbs_moms wait until they get their MOM hierarchy file from pbs_server to send their first update, or until 10 minutes pass. This reduces network traffic on startup and can bring up clusters faster. .IP "\-x" 16 Disables the check for privileged port resource monitor connections. This is used mainly for testing since the privileged port is the only From 9d123c59d71fbf94c31035dfcce76b7c37bf1d2b Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 9 Jan 2017 16:00:59 -0700 Subject: [PATCH 121/312] TRQ-3903 && TRQ-3862. Fix some flags being ignored if they're in the job script. This was a bug introduced by the fix to TRQ-3862, which was too broad and has now been narrowed down a bit. --- src/include/u_hash_map_structs.h | 2 + src/lib/Libcmds/add_verify_resources.c | 2 +- src/lib/Libutils/u_hash_map_structs.c | 42 ++++++++++++++------- src/test/add_verify_resources/scaffolding.c | 2 +- src/test/u_hash_map_structs/test_uut.c | 35 +++++++++-------- 5 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src/include/u_hash_map_structs.h b/src/include/u_hash_map_structs.h index ca6db3ceb5..ca050bcc5c 100644 --- a/src/include/u_hash_map_structs.h +++ b/src/include/u_hash_map_structs.h @@ -123,6 +123,8 @@ typedef job_data_container::item_iterator job_data_iterator; int hash_add_item(job_data_container *head, const char *name, const char *value, int var_type, int op_type); +void hash_priority_add_or_exit(job_data_container *head, const char *name, const char *value, int var_type); + void hash_add_or_exit(job_data_container *head, const char *name, const char *value, int var_type); int hash_del_item(job_data_container *head, const char *name); diff --git a/src/lib/Libcmds/add_verify_resources.c b/src/lib/Libcmds/add_verify_resources.c index e7f0da4dc8..04be1a0f34 100644 --- a/src/lib/Libcmds/add_verify_resources.c +++ b/src/lib/Libcmds/add_verify_resources.c @@ -290,7 +290,7 @@ int add_verify_resources( else snprintf(value, vlen, "%s", v); - hash_add_or_exit(res_attr, name, value, p_type); + hash_priority_add_or_exit(res_attr, name, value, p_type); } else { diff --git a/src/lib/Libutils/u_hash_map_structs.c b/src/lib/Libutils/u_hash_map_structs.c index 525c32a1f0..9e3c37f91e 100644 --- a/src/lib/Libutils/u_hash_map_structs.c +++ b/src/lib/Libutils/u_hash_map_structs.c @@ -107,28 +107,42 @@ int hash_add_item( { job_data *item = new job_data(name,value,var_type,op_type); - bool replace = false; head->lock(); + int ret = head->insert(item,name,true); + head->unlock(); + return ret; + } /* END hash_add_item() */ + + + +void hash_priority_add_or_exit( + + job_data_container *head, + const char *name, + const char *value, + int var_type) + + { + bool should_add = true; job_data *old_item = head->find(name); - int ret = 1; if (old_item != NULL) { - // Only call insert if we are replacing the old item - if (var_type < old_item->var_type) - ret = head->insert(item, name, true); - else - delete item; + // Only call insert if we have priority over the old item + if (var_type >= old_item->var_type) + should_add = false; } - else - ret = head->insert(item, name, replace); - - head->unlock(); - - return ret; - } /* END hash_add_item() */ + if (should_add) + { + if (hash_add_item(head, name, value, var_type, SET) == FALSE) + { + fprintf(stderr, "Error allocating memory for hash (%s)-(%s)\n", name, value); + exit(1); + } + } + } // END hash_priority_add_or_exit() diff --git a/src/test/add_verify_resources/scaffolding.c b/src/test/add_verify_resources/scaffolding.c index a4b2bbdd5c..671f380a0c 100644 --- a/src/test/add_verify_resources/scaffolding.c +++ b/src/test/add_verify_resources/scaffolding.c @@ -4,7 +4,7 @@ #include "u_hash_map_structs.h" /* job_data */ -void hash_add_or_exit(job_data_container *head, const char *name, const char *value, int var_type) +void hash_priority_add_or_exit(job_data_container *head, const char *name, const char *value, int var_type) { fprintf(stderr, "The call to hash_add_or_exit needs to be mocked!!\n"); } diff --git a/src/test/u_hash_map_structs/test_uut.c b/src/test/u_hash_map_structs/test_uut.c index a33cf0d497..6f11c84768 100644 --- a/src/test/u_hash_map_structs/test_uut.c +++ b/src/test/u_hash_map_structs/test_uut.c @@ -25,6 +25,25 @@ void callocVal(char **dest, const char *src) strcpy(*dest, src); } + +START_TEST(test_hash_priority_add_or_exit) + { + job_data_container map; + + hash_priority_add_or_exit(&map, "planet", "sel", CMDLINE_DATA); + hash_priority_add_or_exit(&map, "planet", "nalthis", SCRIPT_DATA); + + job_data *stored; + fail_unless(hash_find(&map, "planet", &stored) == 1); + fail_unless(stored->value == "sel"); // command line is higher priority than script + + hash_priority_add_or_exit(&map, "planet", "roshar", ENV_DATA); + fail_unless(hash_find(&map, "planet", &stored) == 1); + fail_unless(stored->value == "roshar"); // environment data is higher priority than command line + } +END_TEST + + START_TEST(test_hash_add_item_new_count_clear) { int rc = FALSE; @@ -128,8 +147,6 @@ START_TEST(test_hash_print) rc = hash_clear(&the_map); } END_TEST - - /* Testing this involves forcing this to exit - causing a failure. Don't test. START_TEST(test_add_or_exit) { @@ -185,19 +202,6 @@ START_TEST(test_hash_add_hash) fail_unless(rc == 4, "4 items should exist in the hash_map, %d do", rc); fail_unless(hash_find(&map1, "name4", &tmp_item) == TRUE, "hash item name4 not found"); fail_unless(strcmp(tmp_item->value.c_str(), "value6") == 0, "value for name4 should have been value6, but was %s", tmp_item->value.c_str()); - - // Make sure we don't update the lower priority value - rc = hash_add_item(&map1, "name1", "bob", SCRIPT_DATA, SET); - fail_unless(rc == 1, "We should get success even though we didn't update lower priority data"); - job_data *stored; - fail_unless(hash_find(&map1, "name1", &stored) == 1); - fail_unless(stored->value == "value1"); - - fail_unless(hash_add_item(&map1, "planet", "sel", SCRIPT_DATA, SET) == 1); - fail_unless(hash_add_item(&map1, "planet", "roshar", ENV_DATA, SET) == 1); - // Make sure the value is now roshar - it is higher priority - fail_unless(hash_find(&map1, "planet", &stored) == 1); - fail_unless(stored->value == "roshar"); } END_TEST @@ -211,6 +215,7 @@ Suite *u_hash_map_structs_suite(void) tcase_add_test(tc_core, test_hash_print); /*tcase_add_exit_test(tc_core, test_add_or_exit, 1);*/ tcase_add_test(tc_core, test_hash_add_hash); + tcase_add_test(tc_core, test_hash_priority_add_or_exit); suite_add_tcase(s, tc_core); return s; From ae9959792a140c9055801c2d8130e7cefa630621 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 10 Jan 2017 16:18:04 -0700 Subject: [PATCH 122/312] TRQ-3839. Reject invalid memory submissions at submission time. Cgroups do not allow swap limits to be set lower than physical memory limits, so reject these jobs at submission time for the -l syntax as well since these jobs will never run successfully. --- src/cmds/qsub_functions.c | 86 +++++++++++++++++++++++++++ src/include/req.hpp | 3 + src/test/qsub_functions/scaffolding.c | 25 ++++++++ src/test/qsub_functions/test_uut.c | 19 ++++++ 4 files changed, 133 insertions(+) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index f5a1117acb..7a2bc0ba8c 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -1003,6 +1003,86 @@ bool is_resource_request_valid( +/* + * is_memory_request_valid() + * For cgroups, swap cannot be less than phsyical memory, so reject such jobs. + * + * @param ji (I) - the job information + * @param err_msg (O) - the error message + * @return true if the request is valid or cgroups aren't enabled, false otherwise. + */ + +bool is_memory_request_valid( + + job_info *ji, + std::string &err_msg) + + { + bool valid = true; + +#ifdef PENABLE_LINUX_CGROUPS + job_data_container *resources = ji->res_attr; + job_data *jd; + std::string mem_str; + std::string vmem_str; + unsigned long mem = 0; + unsigned long vmem = 0; + + if (hash_find(resources, "vmem", &jd) == 0) + { + if (hash_find(resources, "pvmem", &jd)) + { + read_mem_value(jd->value.c_str(), vmem); + vmem_str = jd->value; + err_msg = "qsub: requested pvmem='"; + err_msg += vmem_str; + } + } + else + { + read_mem_value(jd->value.c_str(), vmem); + vmem_str = jd->value; + err_msg = "qsub: requested vmem='"; + err_msg += vmem_str; + } + + if (vmem != 0) + { + if (hash_find(resources, "mem", &jd)) + { + read_mem_value(jd->value.c_str(), mem); + mem_str = jd->value; + + if (vmem < mem) + { + err_msg += "' which is less than mem='"; + err_msg += mem_str; + err_msg += "'. If p/vmem is set, it must be greater than or equal to mem.\n"; + valid = false; + } + } + else if (hash_find(resources, "pmem", &jd)) + { + read_mem_value(jd->value.c_str(), mem); + mem_str = jd->value; + + if (vmem < mem) + { + err_msg += "' which is less than pmem='"; + err_msg += mem_str; + err_msg += "'. If p/vmem is set, it must be greater than or equal to pmem.\n"; + valid = false; + } + } + } + +#endif + + return(valid); + } // END is_memory_request_valid() + + + void validate_basic_resourcing( job_info *ji) @@ -1016,6 +1096,12 @@ void validate_basic_resourcing( exit(4); } + if (is_memory_request_valid(ji, err_msg) == false) + { + fprintf(stderr, "%s", err_msg.c_str()); + exit(4); + } + } /* END validate_basic_resourcing() */ diff --git a/src/include/req.hpp b/src/include/req.hpp index 441c64da7c..ba7c634126 100644 --- a/src/include/req.hpp +++ b/src/include/req.hpp @@ -224,4 +224,7 @@ class req void set_task_usage_stats(int task_index, unsigned long cput_used, unsigned long long mem_used); }; + +int read_mem_value(const char *value, unsigned long &parsed); + #endif /* _REQ_HPP */ diff --git a/src/test/qsub_functions/scaffolding.c b/src/test/qsub_functions/scaffolding.c index 0becf9ebbd..f9621be9fd 100644 --- a/src/test/qsub_functions/scaffolding.c +++ b/src/test/qsub_functions/scaffolding.c @@ -29,6 +29,7 @@ bool find_mpp = false; bool find_nodes = false; bool find_size = false; bool validate_path = false; +bool mem_fail = false; int req_val = 0; int global_poll_rc = 0; @@ -49,6 +50,7 @@ char *pbs_geterrmsg(int connect) int hash_find(job_data_container *head, const char *name, job_data **env_var) { + static job_data ev("a", "b", 0, 0); if ((find_mpp == true) && (!strcmp(name, "mppwidth"))) return(1); @@ -61,6 +63,12 @@ int hash_find(job_data_container *head, const char *name, job_data **env_var) else if ((validate_path == true) && (!strcmp(name, "validate_path"))) return(1); + else if (mem_fail == true) + { + ev.value = "1"; + *env_var = &ev; + return(1); + } for (unsigned int i = 0; i < in_hash.size(); i++) { @@ -492,3 +500,20 @@ int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const return(global_poll_rc); } + +int read_mem_value(const char *value, unsigned long &parsed) + { + static int count = 0; + + if (mem_fail) + { + if (count++ % 2 == 0) + parsed = 1; + else + parsed = 2; + } + else + parsed = 0; + + return(0); + } diff --git a/src/test/qsub_functions/test_uut.c b/src/test/qsub_functions/test_uut.c index 93ba24c230..e8072699b3 100644 --- a/src/test/qsub_functions/test_uut.c +++ b/src/test/qsub_functions/test_uut.c @@ -22,8 +22,10 @@ int process_opt_K(job_info *ji, const char *cmd_arg, int data_type); int process_opt_m(job_info *ji, const char *cmd_arg, int data_type); int process_opt_p(job_info *ji, const char *cmd_arg, int data_type); int wait_for_read_ready(int, int); +bool is_memory_request_valid(job_info *ji, std::string &err_msg); extern complete_req cr; +extern bool mem_fail; extern bool submission_string_fail; extern bool added_req; extern bool find_nodes; @@ -50,6 +52,22 @@ bool are_we_forking() } +START_TEST(test_is_memory_request_valid) + { +#ifdef PENABLE_LINUX_CGROUPS + job_info ji; + std::string err; + + mem_fail = false; + fail_unless(is_memory_request_valid(&ji, err) == true); + mem_fail = true; + fail_unless(is_memory_request_valid(&ji, err) == false); +#endif + + } +END_TEST + + START_TEST(test_process_opt_K) { job_info ji; @@ -458,6 +476,7 @@ Suite *qsub_functions_suite(void) tc_core = tcase_create("test_wait_for_read_ready"); tcase_add_test(tc_core, test_wait_for_read_ready); + tcase_add_test(tc_core, test_is_memory_request_valid); suite_add_tcase(s, tc_core); return s; From de123f47b3bcf9c756b8dc7e89227ef851303bb3 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 11 Jan 2017 10:57:05 -0700 Subject: [PATCH 123/312] TRQ-3711. Tweak the fix for this to still fail when the mom times out. --- src/server/svr_movejob.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/server/svr_movejob.c b/src/server/svr_movejob.c index 75456dcd8f..88255be958 100644 --- a/src/server/svr_movejob.c +++ b/src/server/svr_movejob.c @@ -758,7 +758,8 @@ int queue_job_on_mom( if ((pc = PBSD_queuejob(con, my_err, (const char *)job_id, (const char *)job_destin, pqjatr, NULL)) == NULL) { - if (*my_err == PBSE_EXPIRED) + if ((*my_err == PBSE_EXPIRED) || + (*my_err == PBSE_TIMEOUT)) { /* queue job timeout based on pbs_tcp_timeout */ timeout = true; @@ -1224,7 +1225,8 @@ int send_job_over_network_with_retries( my_err, mom_err); - if (rc == LOCUTION_SUCCESS) + if ((rc == LOCUTION_SUCCESS) || + (timeout == true)) break; } /* END for (NumRetries) */ @@ -1418,9 +1420,12 @@ int send_job_work( // get a reply from the mom, in which case mom_err will still be set to PBSE_NONE. Otherwise, // it was not a network error and the node is responding. int fail_count_rc = PBSE_NONE; - if ((rc != PBSE_NONE) && + + if (Timeout == true) + fail_count_rc = PBSE_TIMEOUT; + else if ((rc != PBSE_NONE) && (mom_err == PBSE_NONE)) - fail_count_rc = mom_err; + fail_count_rc = rc; if (node_name != NULL) update_failure_counts(node_name, fail_count_rc); From 03269441efac9aeea49b0e634eb4dfbc4cc4593d Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 11 Jan 2017 23:17:23 +0000 Subject: [PATCH 124/312] TRQ-3866 - make torque.setup more robust --- Makefile.am | 20 +++- torque.setup | 85 ----------------- torque.setup.in | 248 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 86 deletions(-) delete mode 100755 torque.setup create mode 100644 torque.setup.in diff --git a/Makefile.am b/Makefile.am index 911e4e6318..31f8a64869 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,6 +3,23 @@ include $(top_srcdir)/buildutils/config.mk ACLOCAL_AMFLAGS = -I buildutils SUBDIRS = src doc contrib/blcr contrib/init.d contrib/systemd +noinst_SCRIPTS = torque.setup + +edit_scripts = sed \ + -e "s|@exec_prefix[@]|$(exec_prefix)|g" \ + -e "s|@PBS_HOME[@]|$(PBS_SERVER_HOME)|g" + +torque.setup: Makefile + rm -f $@ $@.tmp + my_srcdir=''; \ + test -f ./$@.in || my_srcdir=$(srcdir)/; \ + $(edit_scripts) $${my_srcdir}$@.in >$@.tmp + chmod +x $@.tmp + chmod a-w $@.tmp + mv $@.tmp $@ + +torque.setup: torque.setup.in + EXTRA_DIST = acinclude.m4 \ CHANGELOG \ configure.ac \ @@ -25,7 +42,7 @@ EXTRA_DIST = acinclude.m4 \ README.new_in_51 \ Release_Notes \ run_report.py \ - torque.setup \ + torque.setup.in \ torque.spec \ buildutils/config.mk \ buildutils/determine.distro \ @@ -68,6 +85,7 @@ EXTRA_DIST = acinclude.m4 \ bin_SCRIPTS = pbs-config +CLEANFILES = torque.setup DISTCLEANFILES = pbs-config current_hash MOSTLYCLEANFILES = *.gcda *.gcno *.gcov diff --git a/torque.setup b/torque.setup deleted file mode 100755 index d7ac900b90..0000000000 --- a/torque.setup +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh -# torque.setup - -# USAGE: torque.setup [] - -if [ "$1" = "" ] ; then - echo "USAGE: torque.setup []" - exit 1 - fi - -export PATH=/usr/local/sbin:/usr/local/bin:$PATH - -unset PBSDEBUG -TRQ_CHECK=`pgrep trqauthd` -TRQ_CHECK_RES=$? -if [ "$TRQ_CHECK_RES" = "1" ]; then - TRQ_START=`trqauthd` - TRQ_START_RES=$? - if [ "$TRQ_START_RES" -ne "0" ]; then - echo "trqauthd failed to start!!! exiting setup" - exit 1 - else - echo "trqauthd successfully started" - fi -fi - -if [ "$2" = "" ] ; then - # obtain name obtained from reverse lookup of ip address - HOSTNAME=`hostname` -else - HOSTNAME=$2 -fi - -# enable operator privileges - -USER=$1@$HOSTNAME - -echo "initializing TORQUE (admin: $USER)" - -ps -ef | grep -v grep | grep pbs_server - -if [ "$?" -eq "0" ] ; then - echo "ERROR: pbs_server already running... run 'qterm' to stop pbs_server and rerun" - exit 1; -fi - -pbs_server -t create -# Starting in TORQUE 3.1 the server is multi-threaded. -# We need to pause a second to allow the server to finish coming -# up. If we go to qmgr right away it will fail. -sleep 2 - -ps -ef | grep -v grep | grep pbs_server - -if [ "$?" -ne "0" ] ; then - echo "ERROR: pbs_server failed to start, check syslog and server logs for more information" - exit 1; -fi - -echo set server operators += $USER | qmgr - -if [ "$?" -ne "0" ] ; then - echo "ERROR: cannot set TORQUE admins" - qterm - exit 1; -fi - -echo set server managers += $USER | qmgr - - -qmgr -c 'set server scheduling = true' -qmgr -c 'set server keep_completed = 300' -qmgr -c 'set server mom_job_sync = true' - -# create default queue - -qmgr -c 'create queue batch' -qmgr -c 'set queue batch queue_type = execution' -qmgr -c 'set queue batch started = true' -qmgr -c 'set queue batch enabled = true' -qmgr -c 'set queue batch resources_default.walltime = 1:00:00' -qmgr -c 'set queue batch resources_default.nodes = 1' - -qmgr -c 'set server default_queue = batch' - diff --git a/torque.setup.in b/torque.setup.in new file mode 100644 index 0000000000..9e122956f4 --- /dev/null +++ b/torque.setup.in @@ -0,0 +1,248 @@ +#!/bin/sh +# +# torque.setup - create pbs_server database and default queue +# + +# print out basic usage info + +function usage + { + echo "USAGE: torque.setup [-h|--help] [-d config_path|--config-path=config_path] + [--bin_path=bin_path] [--sbin_path=sbin_path] [--syslog] + [] + + -h|--help Show command help. + + -d config_path|--config-path=CONFIG_PATH Specifies the path of the directory + which is home to the server\'s configuration + files. Default is [$PBS_HOME] + + --prefix=PATH Specifies the path to executables in PATH/bin + and PATH/sbin. Default is [$PREFIX] + + --syslog=true|false Log errors to syslog. Default is false. +" >&2 + } + +# +# logit - write a message to a file descriptor and syslog if desired +# +# usage: logit msg [fd [pri]] +# msg - Message. +# fd - File descriptor to write msg to. Default: 2. +# pri - Syslog priority. Default: user.error. See -p argument listed in logger man page. +# +# Note: message is written to syslog only if WANTSYSLOG is true. +# + +function logit + { + # need a message + [ $# -eq 0 ] && return 1 + + # get the message + msg=$1 + shift + + # get the fd for non-syslog output + if [ $# -gt 0 ]; then + fd=$1 + shift + else + # default + fd=2 + fi + + # write to syslog? + if [ $WANTSYSLOG = true ]; then + if [ $# -gt 0 ]; then + pri=$1 + shift + else + # default + pri=user.error + fi + + # write to syslog + logger -t $PROG -p $pri $msg + rc=$? + + if [ $rc -ne 0 ]; then + echo "$PROG: logger failed with $rc" >&2 + exit $rc + fi + fi + + # write to fd + echo "$msg" >&$fd + + return 0 + } + +PROG=`basename $0` + +PREFIX=@exec_prefix@ +BIN_PATH=$PREFIX/bin +SBIN_PATH=$PREFIX/sbin +PBS_HOME=@PBS_HOME@ +WANTSYSLOG=false + +#export POSIXLY_CORRECT=1 +OPTS=`getopt -o hd: -l config-path: -l prefix: -l syslog: -l help -- "$@"` + +if [ $? != 0 ]; then + logit "getop failed" + usage + exit 1 +fi + +eval set -- "$OPTS" + +while true; do + case "$1" in + -d|--config-path) + + PBS_HOME="$2" + shift 2 + ;; + + --prefix) + + PREFIX="$2" + BIN_PATH=$PREFIX/bin + SBIN_PATH=$PREFIX/sbin + shift 2 + ;; + + --syslog) + + WANTSYSLOG=`echo "$2" | tr '[:upper:]' '[:lower:]'` + shift 2 + ;; + + -h|--help) + + usage + exit 0 + ;; + + --) + shift + break + ;; + + *) + break + ;; + + esac +done + +# now check options + +if [ $# -lt 1 -o $# -gt 2 ]; then + usage + exit 1 +fi + +if [ ! -d "$BIN_PATH" ]; then + logit "$BIN_PATH does not exist." + exit 1 +fi + +if [ ! -d "$SBIN_PATH" ]; then + logit "$SBIN_PATH does not exist." + exit 1 +fi + +if [ "$WANTSYSLOG" != true -a "$WANTSYSLOG" != false ]; then + usage + exit 1 +fi + +# make sure logger command present if syslog desired +if [ $WANTSYSLOG = true ]; then + logger -V &>/dev/null + if [ $? -ne 0 ]; then + WANTSYSLOG=false + logit "cannot find logger command." + exit 1 + fi +fi + +export PATH=$SBIN_PATH:$BIN_PATH:$PATH + +unset PBSDEBUG +pgrep trqauthd >/dev/null + +if [ $? -ne 0 ]; then + trqauthd + if [ $? -ne 0 ]; then + logit "trqauthd failed to start!!! exiting setup" + exit 1 + else + logit "trqauthd successfully started" 1 user.info + fi +fi + +if [ "$2" = "" ] ; then + # obtain name obtained from reverse lookup of ip address + HOSTNAME=`hostname` +else + HOSTNAME=$2 +fi + +# enable operator privileges + +USER=$1@$HOSTNAME + +logit "initializing TORQUE (admin: $USER)" 1 user.info + +pgrep pbs_server >/dev/null + +if [ $? -eq 0 ]; then + logit "pbs_server already running... run 'qterm' to stop pbs_server and rerun" + exit 1 +fi + +pbs_server -t create -d $PBS_HOME +# Starting in TORQUE 3.1 the server is multi-threaded. +# We need to pause a second to allow the server to finish coming +# up. If we go to qmgr right away it will fail. +sleep 2 + +pgrep pbs_server >/dev/null + +if [ $? -ne 0 ]; then + logit "ERROR: pbs_server failed to start, check syslog and server logs for more information" + exit 1 +fi + +# setup admins + +qmgr -c "set server operators += $USER" + +if [ $? -ne 0 ] ; then + logit "ERROR: cannot set TORQUE admins" + qterm + exit 1 +fi + +qmgr -c "set server managers += $USER" + +# setup basic parameters + +qmgr -c 'set server scheduling = true' +qmgr -c 'set server keep_completed = 300' +qmgr -c 'set server mom_job_sync = true' + +# create default queue + +qmgr -c 'create queue batch' +qmgr -c 'set queue batch queue_type = execution' +qmgr -c 'set queue batch started = true' +qmgr -c 'set queue batch enabled = true' +qmgr -c 'set queue batch resources_default.walltime = 1:00:00' +qmgr -c 'set queue batch resources_default.nodes = 1' + +qmgr -c 'set server default_queue = batch' From a729d8c4f1934bee4b12f0996892cce71ac6b7d6 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 12 Jan 2017 10:58:10 -0700 Subject: [PATCH 125/312] TRQ-3830. Add a null check to this code. --- src/server/svr_jobfunc.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index a39cbb58cd..9dc9aa2799 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -3379,14 +3379,17 @@ bool contains_execution_slot_request( std::vector *job_resc = (std::vector *)jb->at_val.at_ptr; - for (size_t i = 0; i < job_resc->size(); i++) + if (job_resc != NULL) { - resource &r = job_resc->at(i); - - if (is_execution_slot_resource(r)) + for (size_t i = 0; i < job_resc->size(); i++) { - execution_slot_request = true; - break; + resource &r = job_resc->at(i); + + if (is_execution_slot_resource(r)) + { + execution_slot_request = true; + break; + } } } From 006e45a5481ecd48771d8547dec43361f21f6606 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 16 Jan 2017 15:25:00 -0700 Subject: [PATCH 126/312] TRQ-3876 - add support for multi-homed systems to torque.setup --- torque.setup.in | 88 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 17 deletions(-) diff --git a/torque.setup.in b/torque.setup.in index 9e122956f4..65da3a0d86 100644 --- a/torque.setup.in +++ b/torque.setup.in @@ -79,6 +79,54 @@ function logit return 0 } + +# return list of ip addresses (one from each interface on this host) +function get_ip_addrs + { + ip addr show | awk '/inet [0-9]+/ {split($2, f, "/"); print f[1]}' + } + +# return hostname of given ip addr +function ip2hostname + { + if [ ! -z "$1" ]; then + getent hosts $1 | awk '{print $2}' + fi + } + +# return list containing the hostname of each interface on this machine +function get_hostnames + { + for ip in `get_ip_addrs`; do + hostname=`ip2hostname $ip` + if [ ! -z "$hostname" ]; then + # add hostname to the list + echo "$hostname" + fi + done + } + +# remove localhost from hostname list if it is not the hostname of this host +function remove_localhost + { + hostname_list="$1" + this_hostname=`hostname` + + for hostname in $hostname_list; do + echo "$hostname" | egrep \^localhost &>/dev/null + if [ $? -eq 0 ]; then + # do not add localhost if it is not the name of this host + if [ "$hostname" != "$this_hostname" ]; then + continue + fi + fi + + # add hostname to the list + echo "$hostname" + done + } + + PROG=`basename $0` PREFIX=@exec_prefix@ @@ -184,17 +232,10 @@ if [ $? -ne 0 ]; then logit "trqauthd successfully started" 1 user.info fi fi - -if [ "$2" = "" ] ; then - # obtain name obtained from reverse lookup of ip address - HOSTNAME=`hostname` -else - HOSTNAME=$2 -fi -# enable operator privileges - -USER=$1@$HOSTNAME +# user and optional hostname +USER=$1 +HOSTNAME=$2 logit "initializing TORQUE (admin: $USER)" 1 user.info @@ -220,15 +261,28 @@ fi # setup admins -qmgr -c "set server operators += $USER" +if [ ! -z "$HOSTNAME" ]; then + # supplied to script by caller + hostname_list=$HOSTNAME +else + # look up hostnames for all interfaces on this host + hostname_list_all=`get_hostnames` -if [ $? -ne 0 ] ; then - logit "ERROR: cannot set TORQUE admins" - qterm - exit 1 + # remove localhost if necessary + hostname_list=`remove_localhost "$hostname_list_all"` fi - -qmgr -c "set server managers += $USER" + +for hostname in $hostname_list; do + for admin_type in operators managers; do + qmgr -c "s s $admin_type += $USER@$hostname" + + if [ $? -ne 0 ] ; then + logit "ERROR: cannot set $USER@$hostname in $admin_type list" + qterm + exit 1 + fi + done +done # setup basic parameters From bcc7ac57f49becb3ab419c5617ae729f8813a50e Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 19 Jan 2017 14:30:46 -0700 Subject: [PATCH 127/312] TRQ-3907 - make "pbsnodes -q" do the right thing --- contrib/pestat-1.0/pestat.c | 2 +- src/cmds/pbsnodes.c | 10 ++-- src/cmds/qalter.c | 2 +- src/cmds/qchkpt.c | 2 +- src/cmds/qdel.c | 2 +- src/cmds/qdisable.c | 2 +- src/cmds/qenable.c | 2 +- src/cmds/qgpumode.c | 2 +- src/cmds/qgpureset.c | 2 +- src/cmds/qhold.c | 2 +- src/cmds/qmgr.c | 2 +- src/cmds/qmove.c | 2 +- src/cmds/qmsg.c | 2 +- src/cmds/qorder.c | 2 +- src/cmds/qrerun.c | 2 +- src/cmds/qrls.c | 2 +- src/cmds/qrun.c | 2 +- src/cmds/qselect.c | 2 +- src/cmds/qsig.c | 2 +- src/cmds/qstart.c | 2 +- src/cmds/qstat.c | 6 +- src/cmds/qstop.c | 2 +- src/cmds/qsub_functions.c | 4 +- src/cmds/qterm.c | 2 +- src/gui/Ccode/xpbs_datadump.c | 2 +- src/include/cmds.h | 2 +- src/include/lib_ifl.h | 1 + src/include/pbs_ifl.h | 1 + src/lib/Libcmds/cnt2server.c | 84 +++++++++++++++------------ src/lib/Libcmds/lib_cmds.h | 2 +- src/lib/Libifl/lib_ifl.h | 1 + src/lib/Libifl/pbsD_connect.c | 53 +++++++++++++---- src/test/cnt2server/scaffolding.c | 10 ++++ src/test/cnt2server/test_uut.c | 15 ++++- src/test/pbsnodes/scaffolding.c | 2 +- src/test/qalter/scaffolding.c | 2 +- src/test/qchkpt/scaffolding.c | 2 +- src/test/qdel/scaffolding.c | 2 +- src/test/qdisable/scaffolding.c | 2 +- src/test/qenable/scaffolding.c | 2 +- src/test/qgpumode/scaffolding.c | 2 +- src/test/qgpureset/scaffolding.c | 2 +- src/test/qhold/scaffolding.c | 2 +- src/test/qmgr/scaffolding.c | 2 +- src/test/qmove/scaffolding.c | 2 +- src/test/qmsg/scaffolding.c | 2 +- src/test/qorder/scaffolding.c | 2 +- src/test/qrerun/scaffolding.c | 2 +- src/test/qrls/scaffolding.c | 2 +- src/test/qrun/scaffolding.c | 2 +- src/test/qselect/scaffolding.c | 2 +- src/test/qsig/scaffolding.c | 2 +- src/test/qstart/scaffolding.c | 2 +- src/test/qstat/scaffolding.c | 2 +- src/test/qstop/scaffolding.c | 2 +- src/test/qsub_functions/scaffolding.c | 2 +- src/test/qterm/scaffolding.c | 2 +- 57 files changed, 171 insertions(+), 108 deletions(-) diff --git a/contrib/pestat-1.0/pestat.c b/contrib/pestat-1.0/pestat.c index d55d02f930..e3dfba3ff3 100644 --- a/contrib/pestat-1.0/pestat.c +++ b/contrib/pestat-1.0/pestat.c @@ -195,7 +195,7 @@ int main( } /* END switch (c) */ } - con = cnt2server(def_server); + con = cnt2server(def_server, false); if (con <= 0) { diff --git a/src/cmds/pbsnodes.c b/src/cmds/pbsnodes.c index 894755bd31..4553b54cf7 100644 --- a/src/cmds/pbsnodes.c +++ b/src/cmds/pbsnodes.c @@ -131,7 +131,7 @@ enum note_flags {unused, set, list, append}; -int quiet = 0; +bool quiet = false; char *progname; @@ -228,7 +228,7 @@ static int set_all_nodeattrs( NULL, &local_errno); - if (rc && !quiet) + if (rc && (!quiet)) { fprintf(stderr, "Error setting note attribute for %s - ", nodename); @@ -252,7 +252,7 @@ static int set_all_nodeattrs( NULL, &local_errno); - if (rc && !quiet) + if (rc && (!quiet)) { fprintf(stderr, "Error setting node power state for %s - ", nodename); @@ -825,7 +825,7 @@ int main( case 'q': - quiet = 1; + quiet = true; break; @@ -963,7 +963,7 @@ int main( exit(1); } - con = cnt2server(specified_server); + con = cnt2server(specified_server, quiet); if (con <= 0) { diff --git a/src/cmds/qalter.c b/src/cmds/qalter.c index 023481cbf5..a43c981582 100644 --- a/src/cmds/qalter.c +++ b/src/cmds/qalter.c @@ -681,7 +681,7 @@ int main( cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qchkpt.c b/src/cmds/qchkpt.c index fca647cfd1..445654322d 100644 --- a/src/cmds/qchkpt.c +++ b/src/cmds/qchkpt.c @@ -50,7 +50,7 @@ int main(int argc, char **argv) /* qchkpt */ cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qdel.c b/src/cmds/qdel.c index b5f43b2f32..909018da66 100644 --- a/src/cmds/qdel.c +++ b/src/cmds/qdel.c @@ -310,7 +310,7 @@ int qdel_main( cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qdisable.c b/src/cmds/qdisable.c index 33124c2191..df8948d4ca 100644 --- a/src/cmds/qdisable.c +++ b/src/cmds/qdisable.c @@ -106,7 +106,7 @@ static void execute( NULL, (char *)"enabled", NULL, (char *)"FALSE", SET }; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_manager_err(ct, MGR_CMD_SET, MGR_OBJ_QUEUE, queue, &attr, NULL, &local_errno); diff --git a/src/cmds/qenable.c b/src/cmds/qenable.c index d57faf83b6..8517de8ba5 100644 --- a/src/cmds/qenable.c +++ b/src/cmds/qenable.c @@ -109,7 +109,7 @@ static void execute( NULL, (char *)"enabled", NULL, (char *)"TRUE", SET }; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_manager_err(ct, MGR_CMD_SET, MGR_OBJ_QUEUE, queue, &attr, NULL, &local_errno); diff --git a/src/cmds/qgpumode.c b/src/cmds/qgpumode.c index 376f85f7ab..6303b1a59e 100644 --- a/src/cmds/qgpumode.c +++ b/src/cmds/qgpumode.c @@ -132,7 +132,7 @@ static void execute( /* The request to change mode */ - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_gpumode_err(ct, node, gpuid, mode, &local_errno); diff --git a/src/cmds/qgpureset.c b/src/cmds/qgpureset.c index 629da1091c..cb07bf09de 100644 --- a/src/cmds/qgpureset.c +++ b/src/cmds/qgpureset.c @@ -144,7 +144,7 @@ static void execute( /* The request to change mode */ - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_gpureset_err(ct, node, gpuid, ecc_perm, ecc_vol, &local_errno); diff --git a/src/cmds/qhold.c b/src/cmds/qhold.c index 0f34e2bad6..cce4be7e87 100644 --- a/src/cmds/qhold.c +++ b/src/cmds/qhold.c @@ -137,7 +137,7 @@ int main( cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qmgr.c b/src/cmds/qmgr.c index 5ee1effc4b..1bde061923 100644 --- a/src/cmds/qmgr.c +++ b/src/cmds/qmgr.c @@ -677,7 +677,7 @@ struct server *make_connection( struct server *svr = NULL; - if ((connection = cnt2server(name)) > 0) + if ((connection = cnt2server(name, false)) > 0) { svr = new_server(); diff --git a/src/cmds/qmove.c b/src/cmds/qmove.c index 6118d5ca8c..977bbdea3e 100644 --- a/src/cmds/qmove.c +++ b/src/cmds/qmove.c @@ -66,7 +66,7 @@ int main(int argc, char **argv) /* qmove */ cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qmsg.c b/src/cmds/qmsg.c index b6c70d9400..9f046e3475 100644 --- a/src/cmds/qmsg.c +++ b/src/cmds/qmsg.c @@ -94,7 +94,7 @@ int main( cnt: - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qorder.c b/src/cmds/qorder.c index b9a7fc7990..947050de87 100644 --- a/src/cmds/qorder.c +++ b/src/cmds/qorder.c @@ -135,7 +135,7 @@ int main( if (pn != NULL) *pn = ':'; /* restore : if it was present */ - connect = cnt2server(svrtmp); + connect = cnt2server(svrtmp, false); if (connect <= 0) { diff --git a/src/cmds/qrerun.c b/src/cmds/qrerun.c index 5edeae6bed..f665252d19 100644 --- a/src/cmds/qrerun.c +++ b/src/cmds/qrerun.c @@ -148,7 +148,7 @@ int main( cnt: - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qrls.c b/src/cmds/qrls.c index b7a2e89ba2..e274c340a3 100644 --- a/src/cmds/qrls.c +++ b/src/cmds/qrls.c @@ -151,7 +151,7 @@ int main( cnt: - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qrun.c b/src/cmds/qrun.c index 17cd441b47..64d13a9ee3 100644 --- a/src/cmds/qrun.c +++ b/src/cmds/qrun.c @@ -155,7 +155,7 @@ static void execute( cnt: - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { char job_id[PBS_MAXSVRJOBID]; diff --git a/src/cmds/qselect.c b/src/cmds/qselect.c index c534c860b0..1b4f509f20 100644 --- a/src/cmds/qselect.c +++ b/src/cmds/qselect.c @@ -514,7 +514,7 @@ int main( } } - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qsig.c b/src/cmds/qsig.c index 52f0f14e47..edcfa9a267 100644 --- a/src/cmds/qsig.c +++ b/src/cmds/qsig.c @@ -89,7 +89,7 @@ int main( cnt: - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qstart.c b/src/cmds/qstart.c index 43cb18dc8b..e257331a55 100644 --- a/src/cmds/qstart.c +++ b/src/cmds/qstart.c @@ -107,7 +107,7 @@ static void execute( NULL, (char *)"started", NULL, (char *)"TRUE", SET }; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_manager_err(ct, MGR_CMD_SET, MGR_OBJ_QUEUE, queue, &attr, NULL, &local_errno); diff --git a/src/cmds/qstat.c b/src/cmds/qstat.c index e6d1e936a5..e11d6209c0 100644 --- a/src/cmds/qstat.c +++ b/src/cmds/qstat.c @@ -3097,7 +3097,7 @@ int run_job_mode( while (retry_count < MAX_RETRIES) { - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); any_failed = -1 * connect; if (connect <= 0 ) @@ -3295,7 +3295,7 @@ int run_queue_mode( while(retry_count < MAX_RETRIES) { - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { @@ -3389,7 +3389,7 @@ int run_server_mode( while (retry_count < MAX_RETRIES) { - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qstop.c b/src/cmds/qstop.c index 1f83a75646..73fb5674ee 100644 --- a/src/cmds/qstop.c +++ b/src/cmds/qstop.c @@ -104,7 +104,7 @@ static void execute( NULL, (char *)"started", NULL, (char *)"FALSE", SET }; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_manager_err(ct, MGR_CMD_SET, MGR_OBJ_QUEUE, queue, &attr, NULL, &local_errno); diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 7a2bc0ba8c..5e24560148 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -2142,7 +2142,7 @@ void bailout(void) printf("Job %s is being deleted\n", new_jobname); - c = cnt2server(server_out); + c = cnt2server(server_out, false); if (c <= 0) { @@ -4752,7 +4752,7 @@ void main_func( } } - sock_num = cnt2server(server_out); + sock_num = cnt2server(server_out, false); if (sock_num <= 0) { diff --git a/src/cmds/qterm.c b/src/cmds/qterm.c index 30795a37bd..41d172df91 100644 --- a/src/cmds/qterm.c +++ b/src/cmds/qterm.c @@ -155,7 +155,7 @@ static void execute( char *errmsg; /* Error message from pbs_terminate */ int local_errno = 0; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { err = pbs_terminate_err(ct, manner, NULL, &local_errno); diff --git a/src/gui/Ccode/xpbs_datadump.c b/src/gui/Ccode/xpbs_datadump.c index b7074b12d1..a7e4e8bfb4 100644 --- a/src/gui/Ccode/xpbs_datadump.c +++ b/src/gui/Ccode/xpbs_datadump.c @@ -1622,7 +1622,7 @@ int main( /* qstat */ if (sigsetjmp(env_alrm, 1) == 0) { alarm(timeout_secs); - myconnection = cnt2server(server_out); + myconnection = cnt2server(server_out, false); } alarm(0); diff --git a/src/include/cmds.h b/src/include/cmds.h index 436868a6a4..f93737386b 100644 --- a/src/include/cmds.h +++ b/src/include/cmds.h @@ -130,7 +130,7 @@ void prt_error(char *, char *, int); extern "C" { void set_attr(struct attrl **, const char *, const char *); -int cnt2server(const char *server); +int cnt2server(const char *server, bool silence); } int set_resources(struct attrl **, char *, int); void prt_job_err(const char *, int, const char *); diff --git a/src/include/lib_ifl.h b/src/include/lib_ifl.h index 27043c0750..007487bcd3 100644 --- a/src/include/lib_ifl.h +++ b/src/include/lib_ifl.h @@ -294,6 +294,7 @@ int parse_daemon_response(long long code, long long len, char *buf); ssize_t send_unix_creds(int sd); #endif int pbs_original_connect(char *server); +int pbs_original_connect_ext(char *server, bool silence); int pbs_disconnect_socket(int socket); int pbs_connect_with_retry(char *server_name_ptr, int retry_seconds); void initialize_connections_table(); diff --git a/src/include/pbs_ifl.h b/src/include/pbs_ifl.h index 1f4bbf5d2a..89eb68cdfd 100644 --- a/src/include/pbs_ifl.h +++ b/src/include/pbs_ifl.h @@ -654,6 +654,7 @@ int pbs_asyrunjob(int c, char *jobid, char *location, char *extend); int pbs_alterjob_async(int connect, char *job_id, struct attrl *attrib, char *extend); int pbs_alterjob(int connect, char *job_id, struct attrl *attrib, char *extend); int pbs_connect(char *server); +int pbs_connect_ext(char *server, bool silence); int pbs_query_max_connections(); char *pbs_default(void); char *pbs_fbserver(void); diff --git a/src/lib/Libcmds/cnt2server.c b/src/lib/Libcmds/cnt2server.c index a9d2bfe850..1a558d84e7 100644 --- a/src/lib/Libcmds/cnt2server.c +++ b/src/lib/Libcmds/cnt2server.c @@ -84,14 +84,15 @@ * * Synopsis: * - * int cnt2server(const char *server) + * int cnt2server(const char *server, bool silence) * * server The name of the server to connect to. A NULL or null string * for the default server. + * silence Suppress stderr? * * Returns: * - * The connection returned by pbs_connect(). + * The connection returned by pbs_connect_ext(). */ #include /* the master config generated by configure */ @@ -133,7 +134,8 @@ extern "C" { int cnt2server( - const char *SpecServer) /* I (optional) */ + const char *SpecServer, /* I (optional) */ + bool silence) /* I (write error messages?) */ { int connect; @@ -207,13 +209,13 @@ int cnt2server( } - /* NOTE: env vars PBS_DEFAULT and PBS_SERVER will be checked and applied w/in pbs_connect() */ + /* NOTE: env vars PBS_DEFAULT and PBS_SERVER will be checked and applied w/in pbs_connect_ext() */ start: - connect = pbs_connect(Server); + connect = pbs_connect_ext(Server, silence); - /* if pbs_connect failed maybe the active server is down. validate the active + /* if pbs_connect_ext failed maybe the active server is down. validate the active server and try again */ if ((connect <= 0) && ((SpecServer == NULL) || (SpecServer[0] == '\0')) && (one_host_only == 0)) { @@ -222,9 +224,9 @@ int cnt2server( if (rc != PBSE_NONE) return(connect); - connect = pbs_connect(valid_server); + connect = pbs_connect_ext(valid_server, silence); - if (connect >= 0) + if ((!silence) && (connect >= 0)) fprintf(stderr, "New active server is %s\n", valid_server); free(valid_server); } @@ -239,22 +241,25 @@ int cnt2server( { case PBSE_BADHOST: - if (Server[0] == '\0') + if (!silence) { - fprintf(stderr, "Cannot resolve default server host '%s' - check server_name file.\n", - pbs_default()); - } - else - { - fprintf(stderr, "Cannot resolve specified server host '%s'.\n", - Server); + if (Server[0] == '\0') + { + fprintf(stderr, "Cannot resolve default server host '%s' - check server_name file.\n", + pbs_default()); + } + else + { + fprintf(stderr, "Cannot resolve specified server host '%s'.\n", + Server); + } } break; case PBSE_NOCONNECTS: - if (thistime == 0) + if ((!silence) && (thistime == 0)) fprintf(stderr, "Too many open connections.\n"); if (cnt2server_retry != 0) @@ -264,13 +269,14 @@ int cnt2server( case PBSE_NOSERVER: - fprintf(stderr, "No default server name - check server_name file.\n"); + if (!silence) + fprintf(stderr, "No default server name - check server_name file.\n"); break; case PBSE_SYSTEM: - if (thistime == 0) + if ((!silence) && (thistime == 0)) fprintf(stderr, "System call failure.\n"); if (cnt2server_retry != 0) @@ -280,7 +286,7 @@ int cnt2server( case PBSE_PERM: - if (thistime == 0) + if ((!silence) && (thistime == 0)) fprintf(stderr, "No Permission.\n"); if (cnt2server_retry != 0) @@ -289,13 +295,15 @@ int cnt2server( break; case PBSE_IFF_NOT_FOUND: - - fprintf(stderr, "Unable to authorize user.\n"); + + if (!silence) + fprintf(stderr, "Unable to authorize user.\n"); break; case PBSE_PROTOCOL: - fprintf(stderr, "protocol failure.\n"); + if (!silence) + fprintf(stderr, "protocol failure.\n"); break; case PBSE_SOCKET_FAULT: @@ -325,15 +333,18 @@ int cnt2server( break; } - connect = pbs_connect(new_server_name); - if (connect <= 0) + connect = pbs_connect_ext(new_server_name, silence); + if (!silence) { - if (thistime == 0) - fprintf(stderr, "Communication failure.\n"); - } - else - { - fprintf(stderr, "New active server is %s\n", new_server_name); + if (connect <= 0) + { + if (thistime == 0) + fprintf(stderr, "Communication failure.\n"); + } + else + { + fprintf(stderr, "New active server is %s\n", new_server_name); + } } free(new_server_name); @@ -344,7 +355,7 @@ int cnt2server( default: - if (thistime == 0) + if ((!silence) && (thistime == 0)) fprintf(stderr, "Communication failure.\n"); if (cnt2server_retry != 0) @@ -378,10 +389,11 @@ int cnt2server( goto start; } - fprintf(stderr, "Cannot connect to default server host '%s' - check pbs_server daemon and/or trqauthd.\n", - pbs_default()); + if (!silence) + fprintf(stderr, "Cannot connect to default server host '%s' - check pbs_server daemon and/or trqauthd.\n", + pbs_default()); } - else + else if (!silence) { fprintf(stderr, "Cannot connect to specified server host '%s'.\n", Server); @@ -402,7 +414,7 @@ int cnt2server( retry: - if (thistime == 0) + if ((!silence) && (thistime == 0)) { fprintf(stderr, "Retrying for %d seconds\n", (int)cnt2server_retry); diff --git a/src/lib/Libcmds/lib_cmds.h b/src/lib/Libcmds/lib_cmds.h index 7c6b38571d..f359a3633e 100644 --- a/src/lib/Libcmds/lib_cmds.h +++ b/src/lib/Libcmds/lib_cmds.h @@ -13,7 +13,7 @@ int check_job_name(char *name, int chk_alpha); /* from file cnt2server.c */ int cnt2server_conf(long retry); -int cnt2server(const char *SpecServer); +int cnt2server(const char *SpecServer, bool silence); /* from file cvtdate.c */ time_t cvtdate(char *datestr); diff --git a/src/lib/Libifl/lib_ifl.h b/src/lib/Libifl/lib_ifl.h index 38f99a4ab1..a380bb77a4 100644 --- a/src/lib/Libifl/lib_ifl.h +++ b/src/lib/Libifl/lib_ifl.h @@ -291,6 +291,7 @@ int parse_daemon_response(long long code, long long len, char *buf); ssize_t send_unix_creds(int sd); #endif int pbs_original_connect(char *server); +int pbs_original_connect_ext(char *server, bool silence); int pbs_disconnect_socket(int socket); int pbs_connect_with_retry(char *server_name_ptr, int retry_seconds); void initialize_connections_table(); diff --git a/src/lib/Libifl/pbsD_connect.c b/src/lib/Libifl/pbsD_connect.c index b4591aa2e1..4fcad0f630 100644 --- a/src/lib/Libifl/pbsD_connect.c +++ b/src/lib/Libifl/pbsD_connect.c @@ -887,6 +887,13 @@ int trq_set_preferred_network_interface( +int pbs_original_connect( + + char *server) /* I (FORMAT: NULL | '\0' | HOSTNAME | HOSTNAME:PORT )*/ + + { + return(pbs_original_connect_ext(server, false)); + } /* returns socket descriptor or negative value (-1) on failure */ @@ -896,9 +903,10 @@ int trq_set_preferred_network_interface( /* NOTE: 0 is not a valid return value */ -int pbs_original_connect( +int pbs_original_connect_ext( - char *server) /* I (FORMAT: NULL | '\0' | HOSTNAME | HOSTNAME:PORT )*/ + char *server, /* I (FORMAT: NULL | '\0' | HOSTNAME | HOSTNAME:PORT )*/ + bool silence) /* I */ { struct sockaddr_in server_addr; @@ -1405,9 +1413,12 @@ int pbs_original_connect( if (rc > 0) tmp_err_msg = pbs_strerror(rc); - fprintf(stderr, - "ERROR: cannot authenticate connection to server \"%s\", errno=%d (%s)\n", - server, rc, tmp_err_msg); + if (!silence) + { + fprintf(stderr, + "ERROR: cannot authenticate connection to server \"%s\", errno=%d (%s)\n", + server, rc, tmp_err_msg); + } } local_errno = PBSE_SOCKET_FAULT; @@ -1429,7 +1440,8 @@ int pbs_original_connect( if (rc != PBSE_NONE) { - fprintf(stderr, "%s\n", err_msg.c_str()); + if (!silence) + fprintf(stderr, "%s\n", err_msg.c_str()); goto cleanup_conn; } } /* END if !use_unixsock */ @@ -1566,12 +1578,10 @@ void print_server_port_to_stderr( } /** - * This is a new version of this function that allows - * connecting to a list of servers. It is backwards - * compatible with the previous version in that it - * will accept a single server name. + * Wrapper function for pbs_connect_ext(). * * @param server_name_ptr A pointer to a server name or server name list. + * @param silence Boolean - print error output? * @returns A file descriptor number. */ @@ -1579,6 +1589,25 @@ int pbs_connect( char *server_name_ptr) /* I (optional) */ + { + return(pbs_connect_ext(server_name_ptr, false)); + } + +/** + * Function connects to a list of servers. It is backwards + * compatible with the previous version in that it + * will accept a single server name. + * + * @param server_name_ptr A pointer to a server name or server name list. + * @param silence Boolean - print error output? + * @returns A file descriptor number. + */ + +int pbs_connect_ext( + + char *server_name_ptr, /* I (optional) */ + bool silence) /* I */ + { int connect = -1; int i, list_len; @@ -1620,7 +1649,7 @@ int pbs_connect( current_name); } - if ((connect = pbs_original_connect(current_name)) >= 0) + if ((connect = pbs_original_connect_ext(current_name, silence)) >= 0) { if (getenv("PBSDEBUG")) { @@ -1629,7 +1658,7 @@ int pbs_connect( return(connect); /* Success, we have a connection, return it. */ } - else + else if (!silence) print_server_port_to_stderr(current_name); } } diff --git a/src/test/cnt2server/scaffolding.c b/src/test/cnt2server/scaffolding.c index fbacf0d372..81da836858 100644 --- a/src/test/cnt2server/scaffolding.c +++ b/src/test/cnt2server/scaffolding.c @@ -7,6 +7,8 @@ int pbs_errno = 0; +extern bool global_silence; + extern "C" { int pbs_connect(char *server_name_ptr) @@ -15,6 +17,14 @@ int pbs_connect(char *server_name_ptr) exit(1); } +int pbs_connect_ext(char *server_name_ptr, bool silence) + { + // set global silence + global_silence = silence; + + return(0); + } + char *pbs_default(void) { fprintf(stderr, "The call to pbs_default needs to be mocked!!\n"); diff --git a/src/test/cnt2server/test_uut.c b/src/test/cnt2server/test_uut.c index 37c1eaaa77..e5ffdc2d48 100644 --- a/src/test/cnt2server/test_uut.c +++ b/src/test/cnt2server/test_uut.c @@ -1,5 +1,6 @@ #include "license_pbs.h" /* See here for the software license */ #include "test_cnt2server.h" +#include "cmds.h" #include #include @@ -7,6 +8,7 @@ #include "pbs_error.h" #include "pbs_ifl.h" +bool global_silence = false; START_TEST(test_cnt2server_conf) { @@ -19,10 +21,17 @@ START_TEST(test_cnt2server_conf) } END_TEST -START_TEST(test_two) +START_TEST(test_cnt2server) { + int rc; + rc = cnt2server("foo", false); + fail_unless(rc == 0); + fail_unless(global_silence == false); + rc = cnt2server("foo", true); + fail_unless(rc == 0); + fail_unless(global_silence == true); } END_TEST @@ -33,8 +42,8 @@ Suite *cnt2server_suite(void) tcase_add_test(tc_core, test_cnt2server_conf); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_cnt2server"); + tcase_add_test(tc_core, test_cnt2server); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/pbsnodes/scaffolding.c b/src/test/pbsnodes/scaffolding.c index 60f052f140..e7830388e1 100644 --- a/src/test/pbsnodes/scaffolding.c +++ b/src/test/pbsnodes/scaffolding.c @@ -41,7 +41,7 @@ int MXMLToXString(mxml_t *E, char **Buf, int *BufSize, int MaxBufSize, char **Ta extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qalter/scaffolding.c b/src/test/qalter/scaffolding.c index 434c6b2f8a..cb0f7d7ff5 100644 --- a/src/test/qalter/scaffolding.c +++ b/src/test/qalter/scaffolding.c @@ -37,7 +37,7 @@ int locate_job(char *job_id, char *parent_server, char *located_server) extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qchkpt/scaffolding.c b/src/test/qchkpt/scaffolding.c index cc73683957..ab4546b636 100644 --- a/src/test/qchkpt/scaffolding.c +++ b/src/test/qchkpt/scaffolding.c @@ -15,7 +15,7 @@ int pbs_disconnect(int connect) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qdel/scaffolding.c b/src/test/qdel/scaffolding.c index 31689eafd8..de9d19b2cc 100644 --- a/src/test/qdel/scaffolding.c +++ b/src/test/qdel/scaffolding.c @@ -34,7 +34,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { return 10; } diff --git a/src/test/qdisable/scaffolding.c b/src/test/qdisable/scaffolding.c index 2f2afc7f61..920999796c 100644 --- a/src/test/qdisable/scaffolding.c +++ b/src/test/qdisable/scaffolding.c @@ -26,7 +26,7 @@ int parse_destination_id(char *destination_in, char **queue_name_out, char **ser extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qenable/scaffolding.c b/src/test/qenable/scaffolding.c index 6e8f90af03..bad698c8b1 100644 --- a/src/test/qenable/scaffolding.c +++ b/src/test/qenable/scaffolding.c @@ -26,7 +26,7 @@ int parse_destination_id(char *destination_in, char **queue_name_out, char **ser extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qgpumode/scaffolding.c b/src/test/qgpumode/scaffolding.c index 03d21b1d39..baffbf500d 100644 --- a/src/test/qgpumode/scaffolding.c +++ b/src/test/qgpumode/scaffolding.c @@ -24,7 +24,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qgpureset/scaffolding.c b/src/test/qgpureset/scaffolding.c index 88f1c933b0..3c71b41b2e 100644 --- a/src/test/qgpureset/scaffolding.c +++ b/src/test/qgpureset/scaffolding.c @@ -24,7 +24,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qhold/scaffolding.c b/src/test/qhold/scaffolding.c index f3b60e3aff..e95d23d71e 100644 --- a/src/test/qhold/scaffolding.c +++ b/src/test/qhold/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qmgr/scaffolding.c b/src/test/qmgr/scaffolding.c index 7da9381b8c..c3dc507392 100644 --- a/src/test/qmgr/scaffolding.c +++ b/src/test/qmgr/scaffolding.c @@ -32,7 +32,7 @@ int pbs_disconnect(int connect) extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qmove/scaffolding.c b/src/test/qmove/scaffolding.c index 4a3418f769..7f75057a86 100644 --- a/src/test/qmove/scaffolding.c +++ b/src/test/qmove/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qmsg/scaffolding.c b/src/test/qmsg/scaffolding.c index 22fa1d5480..dc39370ae0 100644 --- a/src/test/qmsg/scaffolding.c +++ b/src/test/qmsg/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qorder/scaffolding.c b/src/test/qorder/scaffolding.c index 83dbe44059..2d814bc6c0 100644 --- a/src/test/qorder/scaffolding.c +++ b/src/test/qorder/scaffolding.c @@ -31,7 +31,7 @@ char *pbs_default(void) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qrerun/scaffolding.c b/src/test/qrerun/scaffolding.c index bb839b3a97..a6e901fb7a 100644 --- a/src/test/qrerun/scaffolding.c +++ b/src/test/qrerun/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qrls/scaffolding.c b/src/test/qrls/scaffolding.c index 57e67b4334..746229ed3c 100644 --- a/src/test/qrls/scaffolding.c +++ b/src/test/qrls/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qrun/scaffolding.c b/src/test/qrun/scaffolding.c index 6672cba1b6..f209bcfeb0 100644 --- a/src/test/qrun/scaffolding.c +++ b/src/test/qrun/scaffolding.c @@ -33,7 +33,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qselect/scaffolding.c b/src/test/qselect/scaffolding.c index 4035d0c88c..7f1dfc1b51 100644 --- a/src/test/qselect/scaffolding.c +++ b/src/test/qselect/scaffolding.c @@ -27,7 +27,7 @@ int parse_destination_id(char *destination_in, char **queue_name_out, char **ser extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qsig/scaffolding.c b/src/test/qsig/scaffolding.c index c260a56bce..63506a8a27 100644 --- a/src/test/qsig/scaffolding.c +++ b/src/test/qsig/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qstart/scaffolding.c b/src/test/qstart/scaffolding.c index 877cc3ccd1..77af2ab827 100644 --- a/src/test/qstart/scaffolding.c +++ b/src/test/qstart/scaffolding.c @@ -33,7 +33,7 @@ int locate_job(char *job_id, char *parent_server, char *located_server) extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qstat/scaffolding.c b/src/test/qstat/scaffolding.c index 68f7440ecf..8158095b6c 100644 --- a/src/test/qstat/scaffolding.c +++ b/src/test/qstat/scaffolding.c @@ -157,7 +157,7 @@ int MXMLToXString(mxml_t *E, char **Buf, int *BufSize, int MaxBufSize, char **Ta extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { if (connect_success == false) return(-1); diff --git a/src/test/qstop/scaffolding.c b/src/test/qstop/scaffolding.c index c8c4073ad7..0b9d631c57 100644 --- a/src/test/qstop/scaffolding.c +++ b/src/test/qstop/scaffolding.c @@ -32,7 +32,7 @@ int locate_job(char *job_id, char *parent_server, char *located_server) extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qsub_functions/scaffolding.c b/src/test/qsub_functions/scaffolding.c index f9621be9fd..fcae119197 100644 --- a/src/test/qsub_functions/scaffolding.c +++ b/src/test/qsub_functions/scaffolding.c @@ -286,7 +286,7 @@ int pbs_disconnect(int connect) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server to be mocked!!\n"); exit(1); diff --git a/src/test/qterm/scaffolding.c b/src/test/qterm/scaffolding.c index f08197d897..6ca4743fd4 100644 --- a/src/test/qterm/scaffolding.c +++ b/src/test/qterm/scaffolding.c @@ -24,7 +24,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); From 7fbea58258a43ff6d4ebe3dd0447d319c956e81e Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 23 Jan 2017 14:53:23 -0700 Subject: [PATCH 128/312] TRQ-3910 - fix deadlock when modifying queues --- src/server/req_manager.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/server/req_manager.c b/src/server/req_manager.c index ffaf486726..45d8af2129 100644 --- a/src/server/req_manager.c +++ b/src/server/req_manager.c @@ -1477,19 +1477,25 @@ void mgr_queue_set( log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_QUEUE, __func__, log_buf); } - svr_queues.lock(); - iter = svr_queues.get_iterator(); - svr_queues.unlock(); - + // see if all queues or just a single queue should be set if ((*preq->rq_ind.rq_manager.rq_objname == '\0') || (*preq->rq_ind.rq_manager.rq_objname == '@')) { + // all queues + qname = all_quename; allques = TRUE; + + svr_queues.lock(); + iter = svr_queues.get_iterator(); + svr_queues.unlock(); + pque = next_queue(&svr_queues,iter); } else { + // single queue + qname = preq->rq_ind.rq_manager.rq_objname; allques = FALSE; @@ -1500,7 +1506,8 @@ void mgr_queue_set( { req_reject(PBSE_UNKQUE, 0, preq, NULL, NULL); - delete iter; + if (iter != NULL) + delete iter; return; } @@ -1531,7 +1538,8 @@ void mgr_queue_set( que_mutex.unlock(); reply_badattr(rc, bad, plist, preq); - delete iter; + if (iter != NULL) + delete iter; return; } @@ -1551,16 +1559,20 @@ void mgr_queue_set( pque = next_queue(&svr_queues,iter); } /* END while (pque != NULL) */ - /* check the appropriateness of the attributes based on queue type */ - svr_queues.lock(); + // if operating on all queues, get new iterator + if (allques == TRUE) + { + svr_queues.lock(); - delete iter; + delete iter; - iter = svr_queues.get_iterator(); - svr_queues.unlock(); + iter = svr_queues.get_iterator(); + svr_queues.unlock(); - if (allques == TRUE) pque = next_queue(&svr_queues,iter); + } + + /* check the appropriateness of the attributes based on queue type */ while (pque != NULL) { From 0a7fcdeab915e1ad8b6171dab74cde1211a20fd6 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 23 Jan 2017 16:56:25 -0700 Subject: [PATCH 129/312] TRQ-3909. Fix some memory leaks when dumping recovered jobs. --- src/server/job_recov.c | 2 +- src/server/svr_jobfunc.c | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/server/job_recov.c b/src/server/job_recov.c index f811246ec6..b226068119 100644 --- a/src/server/job_recov.c +++ b/src/server/job_recov.c @@ -1499,8 +1499,8 @@ int set_array_job_ids( } else { - job_abt(&pj, "Array job missing array struct, aborting job"); snprintf(log_buf, buflen, "array struct missing for array job %s", pj->ji_qs.ji_jobid); + job_abt(&pj, "Array job missing array struct, aborting job"); return(-1); } } diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index 9dc9aa2799..c341804799 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -1165,7 +1165,6 @@ int svr_setjobstate( changed = true; /* if the state is changing, also update the state counts */ - oldstate = pjob->ji_qs.ji_state; if (oldstate != newstate) @@ -1186,12 +1185,17 @@ int svr_setjobstate( if (has_queue_mutex == FALSE) { - pque = get_jobs_queue(&pjob); - if (pque == NULL) + // On start-up, we call this function even though we don't know the job's queue yet. In + // that case, we don't want this function to fail. + if (pjob->ji_qhdr != NULL) { - sprintf(log_buf, "queue not found for jobid %s", jid.c_str()); - log_err(PBSE_UNKQUE, __func__, log_buf); - return(PBSE_UNKQUE); + pque = get_jobs_queue(&pjob); + if (pque == NULL) + { + sprintf(log_buf, "queue not found for jobid %s", jid.c_str()); + log_err(PBSE_UNKQUE, __func__, log_buf); + return(PBSE_UNKQUE); + } } } else @@ -1277,7 +1281,8 @@ int svr_setjobstate( } else { - if (has_queue_mutex == FALSE) + if ((pque != NULL) && + (has_queue_mutex == FALSE)) unlock_queue(pque, __func__, NULL, LOGLEVEL); } } From f47b4af0cff062ce4e37d3fa173a707808686174 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 19 Jan 2017 14:02:21 -0700 Subject: [PATCH 130/312] TRQ-3909. Fix two uninitialized read errors from valgrind. --- src/lib/Libutils/numa_socket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Libutils/numa_socket.cpp b/src/lib/Libutils/numa_socket.cpp index d5915e5986..f718e2397a 100644 --- a/src/lib/Libutils/numa_socket.cpp +++ b/src/lib/Libutils/numa_socket.cpp @@ -323,7 +323,7 @@ int Socket::get_total_gpus() const int Socket::get_available_gpus() const { - int available_gpus; + int available_gpus = 0; for (size_t i = 0; i < this->chips.size(); i++) available_gpus += this->chips[i].get_available_gpus(); From 60715d0c122d0808c3df83da23111469988a211e Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 23 Jan 2017 17:23:08 -0700 Subject: [PATCH 131/312] TRQ-3909. Tweak a unit test. --- src/test/svr_jobfunc/test_uut.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/svr_jobfunc/test_uut.c b/src/test/svr_jobfunc/test_uut.c index 50ffb0b740..8592768f8b 100644 --- a/src/test/svr_jobfunc/test_uut.c +++ b/src/test/svr_jobfunc/test_uut.c @@ -324,6 +324,8 @@ START_TEST(svr_setjobstate_test) fail_unless(svr_setjobstate(&test_job, JOB_STATE_QUEUED, JOB_SUBSTATE_QUEUED, FALSE) == PBSE_NONE); fail_unless(test_job.ji_wattr[JOB_ATR_exec_host].at_val.at_str != NULL, "exec_host list got removed when it shouldn't have..."); + // We don't use this, but we need a dummy due to protections + test_job.ji_qhdr = (pbs_queue *)0x1; decrement_count = 0; fail_unless(svr_setjobstate(&test_job, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE) == PBSE_NONE); fail_unless(decrement_count == 2); From 5b8c6d15f741ea0e49b59cb569755aba76abc56e Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 24 Jan 2017 11:48:51 -0700 Subject: [PATCH 132/312] TRQ-3873. Prevent req_information from growing indefinitely when jobs fail to start. Conflicts: src/test/node_manager/test_uut.c --- src/server/node_manager.c | 4 +-- src/test/node_manager/test_uut.c | 42 +++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index e8bd41010c..e67ffee97d 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -5684,7 +5684,8 @@ void free_nodes( (pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN1) || (pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN2) || (pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN3) || - (pjob->ji_qs.ji_substate == JOB_SUBSTATE_QUEUED)) + (pjob->ji_qs.ji_substate == JOB_SUBSTATE_QUEUED) || + (pjob->ji_qs.ji_substate == JOB_SUBSTATE_TRNOUT)) { cr->clear_allocations(); } @@ -5710,7 +5711,6 @@ void free_nodes( - struct pbsnode *get_compute_node( char *node_name) diff --git a/src/test/node_manager/test_uut.c b/src/test/node_manager/test_uut.c index a17d4bf7dc..a09eefe8b2 100644 --- a/src/test/node_manager/test_uut.c +++ b/src/test/node_manager/test_uut.c @@ -10,8 +10,9 @@ #include "node_manager.h" #include "test_uut.h" #include "pbs_error.h" -#include "server.h" /* server */ +#include "server.h" #include "json/json.h" +#include "complete_req.hpp" const char *exec_hosts = "napali/0+napali/1+napali/2+napali/50+napali/4+l11/0+l11/1+l11/2+l11/3"; char buf[4096]; @@ -19,6 +20,7 @@ const char *napali = "napali"; const char *l11 = "l11"; struct server server; +void free_nodes(job *pjob, const char *spec); int kill_job_on_mom(const char *job_id, struct pbsnode *pnode); int remove_job_from_node(struct pbsnode *pnode, int internal_job_id); bool node_in_exechostlist(const char *, char *, const char *); @@ -55,6 +57,43 @@ extern bool alloc_br_success; extern bool cray_enabled; +START_TEST(free_nodes_test) + { + job pjob; + +#ifdef PENABLE_LINUX_CGROUPS + complete_req cr; + req r; + allocation a; + pjob.ji_wattr[JOB_ATR_exec_host].at_val.at_str = NULL; + pjob.ji_wattr[JOB_ATR_login_node_id].at_val.at_str = NULL; + pjob.ji_wattr[JOB_ATR_req_information].at_flags = ATR_VFLAG_SET; + pjob.ji_wattr[JOB_ATR_req_information].at_val.at_ptr = &cr; + pjob.ji_wattr[JOB_ATR_cpuset_string].at_val.at_str = strdup("roshar:0-31"); + pjob.ji_wattr[JOB_ATR_cpuset_string].at_flags = ATR_VFLAG_SET; + pjob.ji_wattr[JOB_ATR_memset_string].at_val.at_str = strdup("roshar:0-1"); + pjob.ji_wattr[JOB_ATR_memset_string].at_flags = ATR_VFLAG_SET; + r.record_allocation(a); + cr.add_req(r); + + // We shouldn't free allocations for completed jobs + pjob.ji_qs.ji_substate = JOB_SUBSTATE_COMPLETE; + free_nodes(&pjob, "roshar:ppn=32"); + req &ref1 = cr.get_req(0); + fail_unless(ref1.get_req_allocation_count() == 1); + fail_unless(pjob.ji_wattr[JOB_ATR_cpuset_string].at_val.at_str == NULL); + fail_unless(pjob.ji_wattr[JOB_ATR_memset_string].at_val.at_str == NULL); + + // We should free allocations for jobs that failed to start + pjob.ji_qs.ji_substate = JOB_SUBSTATE_TRNOUT; + free_nodes(&pjob, "roshar:ppn=32"); + req &ref2 = cr.get_req(0); + fail_unless(ref2.get_req_allocation_count() == 0); +#endif + } +END_TEST + + START_TEST(add_job_to_gpu_subnode_test) { gpusubn gn; @@ -960,6 +999,7 @@ Suite *node_manager_suite(void) tc_core = tcase_create("more tests"); tcase_add_test(tc_core, translate_job_reservation_info_to_string_test); tcase_add_test(tc_core, test_initialize_alps_req_data); + tcase_add_test(tc_core, free_nodes_test); suite_add_tcase(s, tc_core); tc_core = tcase_create("even more tests"); From 7c9478babce4b76adcf9fb8ba41e45f836f79f83 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 24 Jan 2017 15:58:06 -0700 Subject: [PATCH 133/312] TRQ-3890. Fix routing queues' routing order. It should be a FIFO, and it would've been except for some bugs in container. --- src/include/container.hpp | 20 ++++++++++---------- src/server/job_attr_def.c | 2 +- src/server/svr_movejob.c | 3 ++- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/include/container.hpp b/src/include/container.hpp index 89d77445cc..622d50d103 100644 --- a/src/include/container.hpp +++ b/src/include/container.hpp @@ -166,8 +166,8 @@ class item_container #endif pContainer = pCtner; iter = -1; - pContainer->initialize_ra_iterator(&iter); reversed = reverse; + pContainer->initialize_ra_iterator(&iter, this->reversed); endHit = false; } void reset(void) //Reset the iterator; @@ -183,7 +183,7 @@ class item_container } #endif iter = -1; - pContainer->initialize_ra_iterator(&iter); + pContainer->initialize_ra_iterator(&iter, this->reversed); endHit = false; } private: @@ -299,7 +299,7 @@ class item_container return false; int iter = -1; - initialize_ra_iterator(&iter); + initialize_ra_iterator(&iter, false); while(index--) { item *pItem = next_thing(&iter); @@ -710,11 +710,7 @@ class item_container next = slots[index].next; slots[rc].next = next; slots[index].next = rc; - - if (next != 0) - { - slots[next].prev = rc; - } + slots[next].prev = rc; /* update the last index if needed */ if (last == index) @@ -1050,10 +1046,14 @@ class item_container */ void initialize_ra_iterator( - int *iter) + int *iter, + bool reversed) { - *iter = slots[ALWAYS_EMPTY_INDEX].next; + if (reversed == false) + *iter = slots[ALWAYS_EMPTY_INDEX].next; + else + *iter = slots[ALWAYS_EMPTY_INDEX].prev; } /* END initialize_ra_iterator() */ diff --git a/src/server/job_attr_def.c b/src/server/job_attr_def.c index c46e0b26cc..495767ec8d 100644 --- a/src/server/job_attr_def.c +++ b/src/server/job_attr_def.c @@ -654,7 +654,7 @@ attribute_def job_attr_def[] = ATR_TYPE_LONG, PARENT_TYPE_JOB }, - /* JOB_ATR_queuerank */ + /* JOB_ATR_qrank */ { (char *)ATTR_qrank, /* "queue_rank" */ decode_l, encode_l, diff --git a/src/server/svr_movejob.c b/src/server/svr_movejob.c index 88255be958..95a2f79d35 100644 --- a/src/server/svr_movejob.c +++ b/src/server/svr_movejob.c @@ -371,7 +371,8 @@ int local_move( snprintf(pjob->ji_qs.ji_queue, sizeof(pjob->ji_qs.ji_queue), "%s", destination); - pjob->ji_wattr[JOB_ATR_qrank].at_val.at_long = ++queue_rank; + if (!(pjob->ji_wattr[JOB_ATR_qrank].at_flags & ATR_VFLAG_SET)) + pjob->ji_wattr[JOB_ATR_qrank].at_val.at_long = ++queue_rank; if ((*my_err = svr_enquejob(pjob, FALSE, NULL, reservation, false)) == PBSE_JOB_RECYCLED) return(-1); From e0984779aff89965ffbab4bc7f515ec7242437fe Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 24 Jan 2017 16:32:57 -0700 Subject: [PATCH 134/312] TRQ-3890. Add a unit test. --- src/include/container.hpp | 1 + src/test/job_container/test_uut.c | 40 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/include/container.hpp b/src/include/container.hpp index 622d50d103..681ae41bb9 100644 --- a/src/include/container.hpp +++ b/src/include/container.hpp @@ -664,6 +664,7 @@ class item_container /* update the last index */ slots[last].next = rc; last = rc; + slots[ALWAYS_EMPTY_INDEX].prev = last; /* update the new item's next index */ slots[rc].next = ALWAYS_EMPTY_INDEX; diff --git a/src/test/job_container/test_uut.c b/src/test/job_container/test_uut.c index 96ddfba863..e9e9997fd4 100644 --- a/src/test/job_container/test_uut.c +++ b/src/test/job_container/test_uut.c @@ -11,6 +11,45 @@ void log_err(int,const char *,const char *) {} +START_TEST(iterator_test) + { + all_jobs alljobs; + struct job *jobs[5]; + + for (int i = 0; i < 5; i++) + { + jobs[i] = job_alloc(); + sprintf(jobs[i]->ji_qs.ji_jobid, "%d.threnody", i); + insert_job(&alljobs, jobs[i]); + } + + // get a reverse iterator + all_jobs_iterator *iter = alljobs.get_iterator(true); + for (int i = 4; i >= 0; i--) + { + char buf[1024]; + job *ptr = iter->get_next_item(); + sprintf(buf, "%d.threnody", i); + fail_unless(ptr != NULL); + fail_unless(!strcmp(ptr->ji_qs.ji_jobid, buf), + "Expected %s, but got %s", buf, ptr->ji_qs.ji_jobid); + } + + // get a forwards iterator + iter = alljobs.get_iterator(); + for (int i = 0; i < 5; i++) + { + char buf[1024]; + job *ptr = iter->get_next_item(); + sprintf(buf, "%d.threnody", i); + fail_unless(ptr != NULL); + fail_unless(!strcmp(ptr->ji_qs.ji_jobid, buf), + "Expected %s, but got %s", buf, ptr->ji_qs.ji_jobid); + } + } +END_TEST + + START_TEST(get_correct_jobname_test) { // with nothing set, get_correct_jobname should just return the jobid passed in. @@ -296,6 +335,7 @@ Suite *job_container_suite(void) tc_core = tcase_create("find_job_by_array_with_removed_record"); tcase_add_test(tc_core, find_job_by_array_with_removed_record_test); + tcase_add_test(tc_core, iterator_test); suite_add_tcase(s, tc_core); return(s); From 994387d073f375a330a0d80e19a36b3eebbe6ac2 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 24 Jan 2017 16:40:17 -0700 Subject: [PATCH 135/312] TRQ-3839. Make the error message more precise. --- src/cmds/qsub_functions.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 5e24560148..0e791462a3 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -1057,7 +1057,7 @@ bool is_memory_request_valid( { err_msg += "' which is less than mem='"; err_msg += mem_str; - err_msg += "'. If p/vmem is set, it must be greater than or equal to mem.\n"; + err_msg += "'. If p/vmem is set to a non-zero value, it must be greater than or equal to mem.\n"; valid = false; } } @@ -1070,7 +1070,7 @@ bool is_memory_request_valid( { err_msg += "' which is less than pmem='"; err_msg += mem_str; - err_msg += "'. If p/vmem is set, it must be greater than or equal to pmem.\n"; + err_msg += "'. If p/vmem is set to a non-zero value, it must be greater than or equal to pmem.\n"; valid = false; } } From cab6294476834864d5885bb9e0b1706505337044 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 30 Jan 2017 09:17:59 -0700 Subject: [PATCH 136/312] Fix a configure.ac message. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index bf86fb23a4..0dbbf1fe73 100644 --- a/configure.ac +++ b/configure.ac @@ -767,7 +767,7 @@ fi dnl if using gcc, we can be very strict AC_ARG_ENABLE(gcc_warnings, [ --disable-gcc-warnings Disable gcc strictness and warnings. If using - gcc, default is to not error on any warnings]) + gcc, default is to error on all warnings]) if test "x$GCC" = "xyes" ;then AC_MSG_CHECKING([whether to disable strict gcc warnings]) if test "${enable_gcc_warnings}" = "no" ; then From 741c01e7945001970b94504e090022ca3938fdbf Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 3 Feb 2017 16:56:36 -0700 Subject: [PATCH 137/312] TRQ-3923. Fix a bug in handling job scripts > 64k in size. --- src/server/req_quejob.c | 184 ++++++++++++++++++++++++++-------------- 1 file changed, 118 insertions(+), 66 deletions(-) diff --git a/src/server/req_quejob.c b/src/server/req_quejob.c index 12595c8470..b2a8acb681 100644 --- a/src/server/req_quejob.c +++ b/src/server/req_quejob.c @@ -1993,76 +1993,36 @@ int req_jobcredential( /* - * req_jobscript - receive job script section + * write_job_file() * - * Each section is appended to the file + * @param pj - the job whose file we're writing + * @param append_only - we should be only appending the job file at this time + * @return PBSE_NONE on SUCCESS, false otherwise */ -int req_jobscript( +int write_job_file( batch_request *preq, - bool perform_commit) + job *pj, + std::string &errbuf, + bool append_only) { - int fds; - char namebuf[MAXPATHLEN]; - job *pj; - int filemode = 0600; - char log_buf[LOCAL_LOG_BUF_SIZE]; - int rc = PBSE_NONE; + int filemode = 0600; + int fds = -1; + char namebuf[MAXPATHLEN]; + char log_buf[LOCAL_LOG_BUF_SIZE]; + int rc = PBSE_NONE; std::string adjusted_path_jobs; - errno = 0; - - pj = locate_new_job(preq->rq_ind.rq_jobfile.rq_jobid); - - if (pj == NULL) - { - rc = PBSE_IVALREQ; - snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "cannot locate new job %s (%d - %s)", - preq->rq_ind.rq_jobfile.rq_jobid, errno, strerror(errno)); - log_err(rc, __func__, log_buf); - req_reject(rc, 0, preq, NULL, log_buf); - return(rc); - } - - mutex_mgr job_mutex(pj->ji_mutex, true); - - /* what is the difference between JOB_SUBSTATE_TRANSIN and TRANSICM? */ - if (pj->ji_qs.ji_substate != JOB_SUBSTATE_TRANSIN) - { - rc = PBSE_IVALREQ; - if (errno == 0) - { - snprintf(log_buf, sizeof(log_buf), - "job %s in unexpected state '%s'", - pj->ji_qs.ji_jobid, - PJobSubState[pj->ji_qs.ji_substate]); - } - else - { - snprintf(log_buf, sizeof(log_buf), - "job %s in unexpected state '%s' (errno=%d - %s)", - pj->ji_qs.ji_jobid, - PJobSubState[pj->ji_qs.ji_substate], - errno, - strerror(errno)); - } - - log_err(rc, __func__, log_buf); - req_reject(rc, 0, preq, NULL, log_buf); - return(rc); - } - if (svr_authorize_jobreq(preq, pj) == -1) { rc = PBSE_PERM; snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "cannot authorize request %s (%d-%s)", preq->rq_ind.rq_jobfile.rq_jobid, errno, strerror(errno)); - log_err(rc, __func__, log_buf); - req_reject(rc, 0, preq, NULL, log_buf); - return rc; + errbuf = log_buf; + return(rc); } // get adjusted path_jobs path @@ -2070,15 +2030,16 @@ int req_jobscript( snprintf(namebuf, sizeof(namebuf), "%s%s%s", adjusted_path_jobs.c_str(), pj->ji_qs.ji_fileprefix, JOB_SCRIPT_SUFFIX); - if (pj->ji_qs.ji_un.ji_newt.ji_scriptsz == 0) + if ((append_only == true) || + (pj->ji_qs.ji_un.ji_newt.ji_scriptsz != 0)) { - /* NOTE: fail is job script already exists */ - - fds = open(namebuf, O_WRONLY | O_CREAT | O_EXCL | O_Sync, filemode); + // Open existing file + fds = open(namebuf, O_WRONLY | O_APPEND | O_Sync, filemode); } else { - fds = open(namebuf, O_WRONLY | O_APPEND | O_Sync, filemode); + // Open a new file + fds = open(namebuf, O_WRONLY | O_CREAT | O_EXCL | O_Sync, filemode); } if (fds < 0) @@ -2089,8 +2050,7 @@ int req_jobscript( errno, strerror(errno), msg_script_open); - log_err(rc, __func__, log_buf); - req_reject(rc, 0, preq, NULL, log_buf); + errbuf = log_buf; return rc; } @@ -2105,21 +2065,113 @@ int req_jobscript( errno, strerror(errno), msg_script_write); - log_err(rc, __func__, log_buf); - req_reject(PBSE_INTERNAL, 0, preq, NULL, log_buf); + errbuf = log_buf; close(fds); return rc; } close(fds); - pj->ji_qs.ji_un.ji_newt.ji_scriptsz += preq->rq_ind.rq_jobfile.rq_size; + if (append_only == false) + pj->ji_qs.ji_un.ji_newt.ji_scriptsz += preq->rq_ind.rq_jobfile.rq_size; /* job has a script file */ - pj->ji_qs.ji_svrflags = (pj->ji_qs.ji_svrflags & ~JOB_SVFLG_CHECKPOINT_FILE) | JOB_SVFLG_SCRIPT; + return(PBSE_NONE); + } // END write_job_file() + + + +/* + * req_jobscript - receive job script section + * + * Each section is appended to the file + */ + +int req_jobscript( + + batch_request *preq, + bool perform_commit) + + { + job *pj; + char log_buf[LOCAL_LOG_BUF_SIZE]; + int rc = PBSE_NONE; + std::string errbuf; + + errno = 0; + + pj = locate_new_job(preq->rq_ind.rq_jobfile.rq_jobid); + + if (pj == NULL) + { + rc = -1; + + // Due to the condensed queuing process, sometimes jobs can be already committed by the time we + // send the next step of the job file. Check if it's in the job list and append the file if + // so. + if ((pj = svr_find_job(preq->rq_ind.rq_jobfile.rq_jobid, TRUE)) != NULL) + { + mutex_mgr job_mutex(pj->ji_mutex, true); + rc = write_job_file(preq, pj, errbuf, true); + + // Appended the job script + if (rc == PBSE_NONE) + { + reply_ack(preq); + return(rc); + } + } + + if (rc == -1) + { + rc = PBSE_IVALREQ; + snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "cannot locate new job %s (%d - %s)", + preq->rq_ind.rq_jobfile.rq_jobid, errno, strerror(errno)); + log_err(rc, __func__, log_buf); + } + + req_reject(rc, 0, preq, NULL, log_buf); + return(rc); + } + + mutex_mgr job_mutex(pj->ji_mutex, true); + + /* what is the difference between JOB_SUBSTATE_TRANSIN and TRANSICM? */ + if (pj->ji_qs.ji_substate != JOB_SUBSTATE_TRANSIN) + { + rc = PBSE_IVALREQ; + if (errno == 0) + { + snprintf(log_buf, sizeof(log_buf), + "job %s in unexpected state '%s'", + pj->ji_qs.ji_jobid, + PJobSubState[pj->ji_qs.ji_substate]); + } + else + { + snprintf(log_buf, sizeof(log_buf), + "job %s in unexpected state '%s' (errno=%d - %s)", + pj->ji_qs.ji_jobid, + PJobSubState[pj->ji_qs.ji_substate], + errno, + strerror(errno)); + } + + log_err(rc, __func__, log_buf); + req_reject(rc, 0, preq, NULL, log_buf); + return(rc); + } + + if ((rc = write_job_file(preq, pj, errbuf, false)) != PBSE_NONE) + { + log_err(rc, __func__, log_buf); + req_reject(rc, 0, preq, NULL, log_buf); + return(rc); + } + /* SUCCESS */ if (perform_commit == true) { From 0398c7713b86f5b9ee55a3c7eea985882b1404c0 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 6 Feb 2017 11:30:26 -0700 Subject: [PATCH 138/312] TRQ-3924. Upgrade the init.d script to help ensure that pbs_server goes down. --- contrib/init.d/pbs_server.in | 15 ++++++++--- src/cmds/qterm.c | 52 +++++++++++++++++++++++++++++++++--- src/test/qterm/scaffolding.c | 23 ++++++++++++++++ 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/contrib/init.d/pbs_server.in b/contrib/init.d/pbs_server.in index 22f48ff457..4c27357b75 100644 --- a/contrib/init.d/pbs_server.in +++ b/contrib/init.d/pbs_server.in @@ -51,9 +51,13 @@ kill_pbs_server() { return 0; fi - # Note that qterm shouldn't be used since it will - # kill the master pbs_server which may be on another node (in HA mode) - kill -TERM $pid + # Use a qterm that only acts locally + qterm -l + + # If our local qterm couldn't do anything, then send the signal as a last resort + if [ $? -ne 0]; then + kill -TERM $pid + fi # wait for process to end for i in {1..5}; do @@ -62,7 +66,10 @@ kill_pbs_server() { done # waited too long, try again - killproc pbs_server -TERM + qterm -l + if [ $? -ne 0]; then + killproc pbs_server -TERM + fi return $? } diff --git a/src/cmds/qterm.c b/src/cmds/qterm.c index 41d172df91..d05bd42d4e 100644 --- a/src/cmds/qterm.c +++ b/src/cmds/qterm.c @@ -33,6 +33,33 @@ int exitstatus = 0; /* Exit Status */ static void execute(int, const char *); + +/* + * is_local() - tells you if the specified server is local or not + * + * @param active_server - the server that we need to determine if is local or not + * @return true if the specified server is local + */ + +bool is_local( + + const char *active_server) + + { + int err = 0; + char local_hostname[PBS_MAXHOSTNAME]; + pbs_net_t addr = get_hostaddr(&err, active_server); + + if ((gethostname(local_hostname, sizeof(local_hostname)) == -1) || + (get_fullhostname(local_hostname, local_hostname, sizeof(local_hostname), NULL) == -1) || + (addr == 0)) + return(false); + + return(get_hostaddr(&err, local_hostname) == addr); + } + + + int main( int argc, /* I */ @@ -43,12 +70,13 @@ int main( * This routine sends a Server Shutdown request to the batch server. */ - static char opts[] = "t:"; /* See man getopt */ + static char opts[] = "lt:"; /* See man getopt */ int s; /* The execute line option */ static char usage[] = "Usage: qterm [-t quick] [server ...]\n"; char *type = NULL; /* Pointer to the type of termination */ int manner; /* The type of termination */ int errflg = 0; /* Error flag */ + bool local = false; /* Command line options */ @@ -57,6 +85,10 @@ int main( switch (s) { + case 'l': + + local = true; + case 't': type = optarg; @@ -116,8 +148,20 @@ int main( else { /* shutdown default server */ - - execute(manner, ""); + if (local == true) + { + char *active_server; + int port = 0; + if (get_active_pbs_server(&active_server, &port) == PBSE_NONE) + { + if (is_local(active_server)) + execute(manner, active_server); + else + exitstatus = -1; + } + } + else + execute(manner, ""); } exit(exitstatus); @@ -146,7 +190,7 @@ int main( static void execute( - int manner, /* I */ + int manner, /* I */ const char *server) /* I */ { diff --git a/src/test/qterm/scaffolding.c b/src/test/qterm/scaffolding.c index 6ca4743fd4..be30823c15 100644 --- a/src/test/qterm/scaffolding.c +++ b/src/test/qterm/scaffolding.c @@ -1,6 +1,8 @@ #include "license_pbs.h" /* See here for the software license */ #include #include /* fprintf */ +#include "pbs_error.h" +#include "net_connect.h" int pbs_errno = 0; @@ -44,3 +46,24 @@ int pbs_terminate_err(int c, int manner, char *extend, int *local_errno) } void initialize_network_info() {} + +int get_fullhostname( + + char *shortname, /* I */ + char *namebuf, /* O */ + int bufsize, /* I */ + char *EMsg) /* O (optional,minsize=MAXLINE - 1024) */ + + { + return(PBSE_NONE); + } + +pbs_net_t get_hostaddr( + + int *local_errno, /* O */ + const char *hostname) /* I */ + + { + return(0); + } + From 15c4f70a0c821e80dc7e2a6e5e86a199d8a316b5 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 7 Feb 2017 11:22:19 -0700 Subject: [PATCH 139/312] Fix a scaffolding issue. --- src/test/qterm/scaffolding.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test/qterm/scaffolding.c b/src/test/qterm/scaffolding.c index be30823c15..670f1ff398 100644 --- a/src/test/qterm/scaffolding.c +++ b/src/test/qterm/scaffolding.c @@ -67,3 +67,12 @@ pbs_net_t get_hostaddr( return(0); } +int get_active_pbs_server( + + char **active_server, + int *port) + + { + return(0); + } + From cdd2f477c56084c4bf8f6e8af21d17af49c53d90 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 7 Feb 2017 15:08:02 -0700 Subject: [PATCH 140/312] TRQ-3914. Add start_count to the accounting entry for each job. --- src/server/accounting.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/server/accounting.c b/src/server/accounting.c index d084491cde..a94cc623aa 100644 --- a/src/server/accounting.c +++ b/src/server/accounting.c @@ -217,6 +217,11 @@ int acct_job( sprintf(local_buf, "etime=%ld ", pjob->ji_wattr[JOB_ATR_etime].at_val.at_long); ds += local_buf; + + /* how many times the job has started */ + sprintf(local_buf, "start_count=%ld ", + pjob->get_long_attr(JOB_ATR_start_count)); + ds += local_buf; /* execution start time */ sprintf(local_buf, "start=%ld ", From 724f9acc305dc03f9413c93d097fdaf451c5e46d Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 7 Feb 2017 15:11:00 -0700 Subject: [PATCH 141/312] TRQ-3914. Fix the cherry-pick for this branch. --- src/server/accounting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/accounting.c b/src/server/accounting.c index a94cc623aa..81d6a45d3e 100644 --- a/src/server/accounting.c +++ b/src/server/accounting.c @@ -220,7 +220,7 @@ int acct_job( /* how many times the job has started */ sprintf(local_buf, "start_count=%ld ", - pjob->get_long_attr(JOB_ATR_start_count)); + pjob->ji_wattr[JOB_ATR_start_count].at_val.at_long); ds += local_buf; /* execution start time */ From e5019ed86fd3b4125ed454fe6bea204c00ff7f21 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 7 Feb 2017 15:13:48 -0700 Subject: [PATCH 142/312] TRQ-3805. Reduce the frequency of the "preparing to send 'x' mail" log message. --- src/server/svr_mail.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/server/svr_mail.c b/src/server/svr_mail.c index 7d297a7183..22a6312416 100644 --- a/src/server/svr_mail.c +++ b/src/server/svr_mail.c @@ -627,23 +627,6 @@ void svr_mailowner( return; } - if (LOGLEVEL >= 3) - { - char tmpBuf[LOG_BUF_SIZE]; - - snprintf(tmpBuf, LOG_BUF_SIZE, "preparing to send '%c' mail for job %s to %s (%.64s)\n", - (char)mailpoint, - pjob->ji_qs.ji_jobid, - pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, - (text != NULL) ? text : "---"); - - log_event( - PBSEVENT_ERROR | PBSEVENT_ADMIN | PBSEVENT_JOB, - PBS_EVENTCLASS_JOB, - pjob->ji_qs.ji_jobid, - tmpBuf); - } - /* * if force is true, force the mail out regardless of mailpoint * unless server no_mail_force attribute is set to true @@ -710,6 +693,23 @@ void svr_mailowner( } } + if (LOGLEVEL >= 3) + { + char tmpBuf[LOG_BUF_SIZE]; + + snprintf(tmpBuf, LOG_BUF_SIZE, "preparing to send '%c' mail for job %s to %s (%.64s)\n", + (char)mailpoint, + pjob->ji_qs.ji_jobid, + pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, + (text != NULL) ? text : "---"); + + log_event( + PBSEVENT_ERROR | PBSEVENT_ADMIN | PBSEVENT_JOB, + PBS_EVENTCLASS_JOB, + pjob->ji_qs.ji_jobid, + tmpBuf); + } + /* Who does the mail go to? If mail-list, them; else owner */ mailto[0] = '\0'; From 87fda90a159f5e3ff68da39831044ec8ea04888c Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 7 Feb 2017 15:18:25 -0700 Subject: [PATCH 143/312] TRQ-3811. Add commit hash to pbs_mom --about --- src/resmom/mom_main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index 0e82028a69..bf22c894d4 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -4247,6 +4247,7 @@ void parse_command_line( printf("installdir: %s\n", PBS_INSTALL_DIR); printf("serverhome: %s\n", PBS_SERVER_HOME); printf("version: %s\n", PACKAGE_VERSION); + printf("Commit: %s\n", GIT_HASH); exit(0); } From 4bf87d6e29c1e15338639dc6679aaeed2b925eb3 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 7 Feb 2017 15:20:37 -0700 Subject: [PATCH 144/312] TRQ-3812. Make client commands --version output to stdout, not stderr. --- src/cmds/pbsnodes.c | 2 +- src/cmds/qstat.c | 2 +- src/cmds/qsub_functions.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cmds/pbsnodes.c b/src/cmds/pbsnodes.c index 4553b54cf7..633351454b 100644 --- a/src/cmds/pbsnodes.c +++ b/src/cmds/pbsnodes.c @@ -911,7 +911,7 @@ int main( if ((optarg != NULL) && !strcmp(optarg, "version")) { - fprintf(stderr, "Version: %s\nCommit: %s\n", + fprintf(stdout, "Version: %s\nCommit: %s\n", PACKAGE_VERSION, GIT_HASH); exit(0); diff --git a/src/cmds/qstat.c b/src/cmds/qstat.c index e11d6209c0..a4b19a810c 100644 --- a/src/cmds/qstat.c +++ b/src/cmds/qstat.c @@ -2817,7 +2817,7 @@ int process_commandline_opts( if ((optarg != NULL) && !strcmp(optarg, "version")) { - fprintf(stderr, "Version: %s\nCommit: %s\n", + fprintf(stdout, "Version: %s\nCommit: %s\n", PACKAGE_VERSION, GIT_HASH); exit(0); diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 0e791462a3..f7c72833e9 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -4360,7 +4360,7 @@ void process_early_opts( TShowAbout_exit(); else if (strcmp(name, "version") == 0) { - fprintf(stderr, "Version: %s\nCommit: %s\n", PACKAGE_VERSION, GIT_HASH); + fprintf(stdout, "Version: %s\nCommit: %s\n", PACKAGE_VERSION, GIT_HASH); exit(0); } } From 4f23f0c9a8748bad2f01874ca0a0c19af3a92cba Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 10 Feb 2017 16:04:58 -0700 Subject: [PATCH 145/312] TRQ-3911. Make interactive jobs more resilient against resetting the port they listen on. --- src/cmds/qsub_functions.c | 139 +++++++++++++++++++++++--------------- 1 file changed, 86 insertions(+), 53 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index f7c72833e9..732ccac326 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -2324,6 +2324,83 @@ int wait_for_read_ready( } + +/* + * get_interactive_job_id() + * + */ + +int get_interactive_job_id( + + int &news, + char *momjobid, + int momjobid_size) + + { + int rc = PBSE_NONE; + torque_socklen_t fromlen; + char cur_server[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2]; + + struct sockaddr_in from; + + // Accept connection on socket set up earlier + while (true) + { + int rc; + + if ((rc = wait_for_read_ready(inter_sock, 30)) < 0) + print_qsub_usage_exit("qsub: poll failed"); + + if (rc > 0) + break; + + /* connect to server, status job to see if still there */ + + if (!locate_job(new_jobname, server_out, cur_server)) + { + fprintf(stderr, "qsub: job %s apparently deleted\n", + new_jobname); + + exit(1); + } + } + + // apparently someone is attempting to connect to us + fromlen = sizeof(from); + + if ((news = accept(inter_sock, (struct sockaddr *) & from, &fromlen)) < 0) + print_qsub_usage_exit("qsub: accept error"); + + // When MOM connects, she will send the job id for us to verify + int amt = momjobid_size; + char *pc = momjobid; + + while (amt > 0) + { + int amt_read = read_ac_socket(news, pc, amt); + + if (amt_read <= 0) + break; + + pc += amt_read; + amt -= amt_read; + + if (*(pc - 1) == '\0') + break; + } + + // Return -1 if we didn't read anything + if (amt == momjobid_size) + { + shutdown(news, 2); + rc = -1; + } + + return(rc); + } // END get_interactive_job_id() + + + /* * interactive - set up for interactive communication with job */ @@ -2332,20 +2409,14 @@ void interactive( job_data_container *client_attr) { - int amt; - char cur_server[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2]; - char momjobid[LOG_BUF_SIZE+1]; int news; - char *pc; struct sigaction act; - struct sockaddr_in from; - torque_socklen_t fromlen; - struct winsize wsz; job_data *tmp_job_info; + bool read_jobid = false; /* Catch SIGINT and SIGTERM, and */ /* setup to catch Death of child */ @@ -2379,59 +2450,21 @@ void interactive( } - printf("qsub: waiting for job %s to start\n", - - new_jobname); - - /* Accept connection on socket set up earlier */ + printf("qsub: waiting for job %s to start\n", new_jobname); - while (true) + for (int retry = 0; retry < 3; retry++) { - int rc; - - if ((rc = wait_for_read_ready(inter_sock, 30)) < 0) - print_qsub_usage_exit("qsub: poll failed"); - - if (rc > 0) - break; - - /* connect to server, status job to see if still there */ - - if (!locate_job(new_jobname, server_out, cur_server)) + if (get_interactive_job_id(news, momjobid, sizeof(momjobid)) == PBSE_NONE) { - fprintf(stderr, "qsub: job %s apparently deleted\n", - new_jobname); - - exit(1); + read_jobid = true; + break; } } - /* apparently someone is attempting to connect to us */ - - fromlen = sizeof(from); - - if ((news = accept(inter_sock, (struct sockaddr *) & from, &fromlen)) < 0) - print_qsub_usage_exit("qsub: accept error"); - - /* When MOM connects, she will send the job id for us to verify */ - - amt = LOG_BUF_SIZE + 1; - - pc = momjobid; - - while (amt > 0) + if (read_jobid == false) { - fromlen = read_ac_socket(news, pc, amt); - - if (fromlen <= 0) - break; - - pc += fromlen; - - if (*(pc - 1) == '\0') - break; - - amt -= fromlen; + fprintf(stderr, "qsub: received 3 connections, but couldn't read a valid job id\n"); + exit(1); } if (strncmp(momjobid, "PBS:", 4) == 0) From 7d61ed311b49e99445bb8f1808a2767361e05c3b Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 10 Feb 2017 16:48:13 -0700 Subject: [PATCH 146/312] TRQ-3929 - don't validate -w path when VALIDATEPATH not set or is false --- src/cmds/qsub_functions.c | 33 ++++++++++++++++----------- src/test/qsub_functions/scaffolding.c | 8 +++++++ src/test/qsub_functions/test_uut.c | 17 ++++++++++++++ 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 732ccac326..815c133642 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -790,16 +790,16 @@ int validate_submit_filter( void validate_pbs_o_workdir( - job_data_container *job_attr) + const job_info *ji) { - job_data *tmp_job_info = NULL; + job_data *tmp_job_info = NULL; const char *the_val = NULL; char tmp_dir[MAXPATHLEN] = {""}; - if (hash_find(job_attr, ATTR_init_work_dir, &tmp_job_info) == FALSE) + if (hash_find(ji->job_attr, ATTR_init_work_dir, &tmp_job_info) == FALSE) { - if (hash_find(job_attr, "PWD", &tmp_job_info)) + if (hash_find(ji->job_attr, "PWD", &tmp_job_info)) the_val = tmp_job_info->value.c_str(); else { @@ -816,19 +816,26 @@ void validate_pbs_o_workdir( } else { - struct stat sb; - + // save the value of the work dir job attribute the_val = tmp_job_info->value.c_str(); - if ((stat(the_val, &sb) != 0) || - (!(S_ISDIR(sb.st_mode)))) + + if (hash_find(ji->client_attr, "validate_path", &tmp_job_info)) { - fprintf(stderr, "qsub: Requested working directory '%s' is not a valid directory\nPlease specify a valid working directory.\n", the_val); - exit(3); + // validate local existence of '-w' working directory + + struct stat sb; + + if ((stat(the_val, &sb) != 0) || + (!(S_ISDIR(sb.st_mode)))) + { + fprintf(stderr, "qsub: Requested working directory '%s' is not a valid directory\nPlease specify a valid working directory.\n", the_val); + exit(3); + } } } - hash_add_or_exit(job_attr, ATTR_pbs_o_workdir, the_val, ENV_DATA); - hash_add_or_exit(job_attr, ATTR_init_work_dir, the_val, ENV_DATA); + hash_add_or_exit(ji->job_attr, ATTR_pbs_o_workdir, the_val, ENV_DATA); + hash_add_or_exit(ji->job_attr, ATTR_init_work_dir, the_val, ENV_DATA); } /* END validate_pbs_o_workdir() */ @@ -1180,7 +1187,7 @@ void validate_join_options ( void post_check_attributes(job_info *ji, char *script_tmp) { - validate_pbs_o_workdir(ji->job_attr); + validate_pbs_o_workdir(ji); validate_qsub_host_pbs_o_server(ji->job_attr); validate_basic_resourcing(ji); diff --git a/src/test/qsub_functions/scaffolding.c b/src/test/qsub_functions/scaffolding.c index fcae119197..d50329969a 100644 --- a/src/test/qsub_functions/scaffolding.c +++ b/src/test/qsub_functions/scaffolding.c @@ -31,6 +31,7 @@ bool find_size = false; bool validate_path = false; bool mem_fail = false; int req_val = 0; +bool init_work_dir = false; int global_poll_rc = 0; short global_poll_revents = 0; @@ -63,6 +64,13 @@ int hash_find(job_data_container *head, const char *name, job_data **env_var) else if ((validate_path == true) && (!strcmp(name, "validate_path"))) return(1); + else if ((init_work_dir == true) && + (!strcmp(name, "init_work_dir"))) + { + // set env_var to point to a legit directory, ".." + *env_var = new job_data(strdup("foo"), strdup(".."), 0, 0); + return(1); + } else if (mem_fail == true) { ev.value = "1"; diff --git a/src/test/qsub_functions/test_uut.c b/src/test/qsub_functions/test_uut.c index e8072699b3..39a18bc7f9 100644 --- a/src/test/qsub_functions/test_uut.c +++ b/src/test/qsub_functions/test_uut.c @@ -23,6 +23,7 @@ int process_opt_m(job_info *ji, const char *cmd_arg, int data_type); int process_opt_p(job_info *ji, const char *cmd_arg, int data_type); int wait_for_read_ready(int, int); bool is_memory_request_valid(job_info *ji, std::string &err_msg); +int validate_pbs_o_workdir(const job_info *); extern complete_req cr; extern bool mem_fail; @@ -32,6 +33,7 @@ extern bool find_nodes; extern bool find_mpp; extern bool find_size; extern bool validate_path; +extern bool init_work_dir; extern std::string added_value; extern std::string added_name; @@ -438,8 +440,22 @@ START_TEST(test_wait_for_read_ready) rc = wait_for_read_ready(some_fd, some_timeout_sec); fail_unless(rc == 0); } +END_TEST + + +START_TEST(test_validate_pbs_o_workdir) + { + job_info ji; + validate_path = true; + + init_work_dir = true; + validate_pbs_o_workdir(&ji); + fail_unless(strcmp(added_name.c_str(), "..") != 0); + } END_TEST + + Suite *qsub_functions_suite(void) { Suite *s = suite_create("qsub_functions methods"); @@ -452,6 +468,7 @@ Suite *qsub_functions_suite(void) tcase_add_test(tc_core, test_process_opt_m); tcase_add_test(tc_core, test_process_opt_p); tcase_add_test(tc_core, test_retry_submit_error); + tcase_add_test(tc_core, test_validate_pbs_o_workdir); suite_add_tcase(s, tc_core); tc_core = tcase_create("test isWindowsFormat"); From afbb77f2c48e81f83a9de139c48862b307730cd0 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 13 Feb 2017 10:59:12 -0700 Subject: [PATCH 147/312] TRQ-3882 - avoid stat'ing mom if job no longer running --- src/server/req_stat.c | 14 ++++++++++++++ src/test/req_stat/scaffolding.c | 4 ++++ src/test/req_stat/test_uut.c | 22 ++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/src/server/req_stat.c b/src/server/req_stat.c index a160dafcec..1181feef3e 100644 --- a/src/server/req_stat.c +++ b/src/server/req_stat.c @@ -747,6 +747,20 @@ int stat_to_mom( mutex_mgr job_mutex(pjob->ji_mutex, true); + // don't continue if job no longer running + if (pjob->ji_qs.ji_state != JOB_STATE_RUNNING) + { + if (LOGLEVEL >= 6) + { + snprintf(log_buf, sizeof(log_buf), + "stat_to_mom(): job is no longer in running state. Not contacting mom."); + + log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_JOB, job_id, log_buf); + } + + return(PBSE_BADSTATE); + } + if ((pjob->ji_qs.ji_un.ji_exect.ji_momaddr == 0) || (!pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str)) { diff --git a/src/test/req_stat/scaffolding.c b/src/test/req_stat/scaffolding.c index 789dcda60e..cbba4d809a 100644 --- a/src/test/req_stat/scaffolding.c +++ b/src/test/req_stat/scaffolding.c @@ -29,6 +29,7 @@ int svr_totnodes = 0; int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ bool exit_called = false; int abort_called; +bool svr_find_job_returns_null = false; struct batch_request *alloc_br(int type) { @@ -247,6 +248,9 @@ int svr_setjobstate(job *pjob, int newstate, int newsubstate, int has_queue_mut job *svr_find_job(const char *jobid, int get_subjob) { job *pjob = (job *)calloc(1, sizeof(job)); + if (svr_find_job_returns_null) + return(NULL); + strcpy(pjob->ji_qs.ji_jobid, jobid); return(pjob); } diff --git a/src/test/req_stat/test_uut.c b/src/test/req_stat/test_uut.c index d95a2dcef1..02bfe26bd3 100644 --- a/src/test/req_stat/test_uut.c +++ b/src/test/req_stat/test_uut.c @@ -7,8 +7,11 @@ #include "array.h" bool in_execution_queue(job *pjob, job_array *pa); +//int stat_to_mom(const char*, struct stat_cntl*); + job *get_next_status_job(struct stat_cntl *cntl, int &job_array_index, job_array *pa, all_jobs_iterator *iter); extern int abort_called; +extern bool svr_find_job_returns_null; enum TJobStatTypeEnum { @@ -106,6 +109,21 @@ START_TEST(test_get_next_status_job) END_TEST +START_TEST(test_stat_to_mom) + { + struct stat_cntl *scp; + + scp = (struct stat_cntl *)calloc(1, sizeof(struct stat_cntl)); + + svr_find_job_returns_null = true; + fail_unless(stat_to_mom(strdup("12345"), scp) == PBSE_JOBNOTFOUND); + + svr_find_job_returns_null = false; + fail_unless(stat_to_mom(strdup("12345"), scp) == PBSE_BADSTATE); + } +END_TEST + + Suite *req_stat_suite(void) { Suite *s = suite_create("req_stat_suite methods"); @@ -118,6 +136,10 @@ Suite *req_stat_suite(void) tcase_add_test(tc_core, test_get_next_status_job); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_stat_to_mom"); + tcase_add_test(tc_core, test_stat_to_mom); + suite_add_tcase(s, tc_core); + return s; } From ed269302f1706a81af2eb4ff9e51d68c7aef7388 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 14 Feb 2017 11:18:54 -0700 Subject: [PATCH 148/312] TRQ-3793. Include the output and error paths in abort emails. --- src/server/svr_mail.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/svr_mail.c b/src/server/svr_mail.c index 22a6312416..5dd8967ba2 100644 --- a/src/server/svr_mail.c +++ b/src/server/svr_mail.c @@ -792,7 +792,7 @@ void svr_mailowner( if (pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str != NULL) mi.jobname = pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str; - if (mailpoint == (int) MAIL_END) + if (mailpoint == (int) MAIL_END || mailpoint == MAIL_ABORT) set_output_files(pjob, &mi); if (text) From 8fe6b91bcba341c081063a3548ac0827331877c2 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 14 Feb 2017 11:25:46 -0700 Subject: [PATCH 149/312] TRQ-3917. Make the release hold error message more specific when it is due to slot limits. --- src/include/pbs_error_db.h | 1 + src/server/req_holdjob.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/include/pbs_error_db.h b/src/include/pbs_error_db.h index 4df4b695c5..3720449f7e 100644 --- a/src/include/pbs_error_db.h +++ b/src/include/pbs_error_db.h @@ -298,6 +298,7 @@ PbsErrClient(PBSE_CGROUP_CREATE_FAIL, (char *)"Could not create all of the cgrou PbsErrClient(PBSE_EOF, (char *)"This stream has already been closed. End of File.") PbsErrClient(PBSE_GPU_PROHIBITED_MODE, (char *)"Invalid gpu mode requested. Prohibited mode is not allowed. Check the spelling of the mode request for errors") PbsErrClient(PBSE_NODE_DELETED, (char *)"Node was deleted during work") +PbsErrClient(PBSE_STATE_SLOT_LIMIT, (char *)"The requested state or substate can't be set from the job's current state (slot limit in effect).") /* pbs client errors ceiling (max_client_err + 1) */ PbsErrClient(PBSE_CEILING, (char*)0) #endif diff --git a/src/server/req_holdjob.c b/src/server/req_holdjob.c index 8d271897a2..ef27d868dc 100644 --- a/src/server/req_holdjob.c +++ b/src/server/req_holdjob.c @@ -459,7 +459,7 @@ int release_job( preq->rq_host); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf); - rc = PBSE_BAD_JOB_STATE_TRANSITION; + rc = PBSE_STATE_SLOT_LIMIT; } return(rc); From 7d0164690c6a33ee145259f57d37d5347481ac8b Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 14 Feb 2017 15:56:16 -0700 Subject: [PATCH 150/312] TRQ-3881. Don't hold the GPUs the entire time pbs_mom is running. --- src/resmom/mom_main.c | 15 ++---------- src/resmom/mom_server_lib.h | 2 +- src/resmom/nvidia.c | 47 +++++++++++++++++++++++++++++++------ 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index bf22c894d4..53a9b540fe 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -299,7 +299,6 @@ extern int check_for_mics(uint32_t& mic_count); #ifdef NVIDIA_GPUS #ifdef NVML_API -extern int init_nvidia_nvml(unsigned int &device_count); extern int shut_nvidia_nvml(); #endif /* NVML_API */ extern int check_nvidia_setup(); @@ -711,10 +710,6 @@ void die( cleanup(); -#if defined(NVIDIA_GPUS) && defined(NVML_API) - shut_nvidia_nvml(); -#endif /* NVIDIA_GPUS and NVML_API */ - log_close(1); exit(1); @@ -3084,10 +3079,6 @@ int rm_request( cleanup(); -#if defined(NVIDIA_GPUS) && defined(NVML_API) - shut_nvidia_nvml(); -#endif /* NVIDIA_GPUS and NVML_API */ - /* We use delete_job_files_sem to make sure there are no outstanding job cleanup routines in progress before we exit. delete_job_files_sem @@ -7128,6 +7119,8 @@ int main( { use_nvidia_gpu = FALSE; } + else + shut_nvidia_nvml(); #endif /* NVML_API */ if (!check_nvidia_setup()) { @@ -7151,10 +7144,6 @@ int main( /* shutdown mom */ -#if defined(NVIDIA_GPUS) && defined(NVML_API) - shut_nvidia_nvml(); -#endif /* NVIDIA_GPUS and NVML_API */ - mom_close_poll(); net_close(-1); /* close all network connections */ diff --git a/src/resmom/mom_server_lib.h b/src/resmom/mom_server_lib.h index 775500cc47..b62204f0e1 100644 --- a/src/resmom/mom_server_lib.h +++ b/src/resmom/mom_server_lib.h @@ -42,7 +42,7 @@ void gen_gen(const char *name, char **BPtr, int *BSpace); #if defined(NVIDIA_GPUS) && defined(NVML_API) void log_nvml_error(nvmlReturn_t rc, char* gpuid, const char* id); -int init_nvidia_nvml(unsigned int &gpu_count); +bool init_nvidia_nvml(unsigned int &gpu_count); int shut_nvidia_nvml(); diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index 628acc9755..a6c152b00f 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -263,7 +263,10 @@ void log_nvml_error( * Function to initialize the Nvidia nvml api */ -int init_nvidia_nvml(unsigned int &device_count) +bool init_nvidia_nvml( + + unsigned int &device_count) + { nvmlReturn_t rc; @@ -275,7 +278,7 @@ int init_nvidia_nvml(unsigned int &device_count) if (rc == NVML_SUCCESS) { if ((int)device_count > 0) - return (TRUE); + return(true); sprintf(log_buffer,"No Nvidia gpus detected\n"); log_ext(-1, __func__, log_buffer, LOG_DEBUG); @@ -284,14 +287,16 @@ int init_nvidia_nvml(unsigned int &device_count) shut_nvidia_nvml(); - return (FALSE); + return(false); } } log_nvml_error (rc, NULL, __func__); - return (FALSE); - } + return(false); + } // END init_nvidia_nvml() + + /* * Function to shutdown the Nvidia nvml api @@ -658,8 +663,12 @@ int setgpumode( nvmlDevice_t device_hndl; char gpu_id[20]; + if (!init_nvidia_nvml(global_gpu_count)) + return(PBSE_GPU_NOT_INITIALIZED); + if (!check_nvidia_setup()) { + shut_nvidia_nvml(); return(PBSE_GPU_NOT_INITIALIZED); } @@ -672,6 +681,7 @@ int setgpumode( compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_THREAD; break; case gpu_prohibited: + shut_nvidia_nvml(); return (PBSE_IVALREQ); break; case gpu_exclusive_process: @@ -684,6 +694,8 @@ int setgpumode( rc); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); } + + shut_nvidia_nvml(); return (PBSE_IVALREQ); } @@ -705,7 +717,10 @@ int setgpumode( rc = nvmlDeviceSetComputeMode(device_hndl, compute_mode); if (rc == NVML_SUCCESS) + { + shut_nvidia_nvml(); return (PBSE_NONE); + } sprintf(gpu_id, "%d", gpuid); log_nvml_error (rc, gpu_id, __func__); @@ -713,6 +728,7 @@ int setgpumode( sprintf(log_buffer, "Failed to get device handle for gpu id %d, nvml error %d", gpuid, rc); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + shut_nvidia_nvml(); return(PBSE_SYSTEM); #else @@ -812,9 +828,13 @@ int resetgpuecc( nvmlEccCounterType_enum counter_type; nvmlDevice_t device_hndl; char gpu_id[20]; + + if (!init_nvidia_nvml(global_gpu_count)) + return(PBSE_GPU_NOT_INITIALIZED); if (!check_nvidia_setup()) { + shut_nvidia_nvml(); return(PBSE_GPU_NOT_INITIALIZED); } @@ -848,7 +868,10 @@ int resetgpuecc( rc = nvmlDeviceClearEccErrorCounts(device_hndl, counter_type); if (rc == NVML_SUCCESS) + { + shut_nvidia_nvml(); return (PBSE_NONE); + } sprintf(gpu_id, "%d", gpuid); log_nvml_error (rc, gpu_id, __func__); @@ -856,6 +879,7 @@ int resetgpuecc( sprintf(log_buffer, "Failed to get device handle for gpu id %d - nvml error %d", gpuid, rc); log_err(-1, __func__, log_buffer); + shut_nvidia_nvml(); return(PBSE_SYSTEM); #else @@ -1262,8 +1286,12 @@ void generate_server_gpustatus_nvml( unsigned long long ecc_counts; char tmpbuf[1024+1]; + if (!init_nvidia_nvml(global_gpu_count)) + return; + if (!check_nvidia_setup()) { + shut_nvidia_nvml(); return; } @@ -1272,7 +1300,10 @@ void generate_server_gpustatus_nvml( #ifdef NUMA_SUPPORT // does this node have gpus configured? if (node_boards[numa_index].gpu_end_index < 0) + { + shut_nvidia_nvml(); return; + } #endif /* get timestamp to report */ @@ -1301,6 +1332,7 @@ void generate_server_gpustatus_nvml( if (rc != NVML_SUCCESS) { log_nvml_error (rc, NULL, __func__); + shut_nvidia_nvml(); return; } #endif @@ -1564,10 +1596,11 @@ void generate_server_gpustatus_nvml( } } + + shut_nvidia_nvml(); return; - - } + } // END generate_server_gpustatus_nvml() #endif /* NVML_API */ From b2ce974ef2f1fb012d927c015eaeb55b0fd6a775 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 15 Feb 2017 17:21:38 -0700 Subject: [PATCH 151/312] TRQ-3932. Fix a race condition for max_user_queuable and array submission. --- src/server/svr_jobfunc.c | 22 +++++++------- src/server/user_info.c | 54 ++++++++++++++++++----------------- src/test/user_info/test_uut.c | 14 ++++++++- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index c341804799..f28fbf3706 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -567,18 +567,18 @@ int svr_enquejob( /* update counts: queue and queue by state */ pque->qu_numjobs++; pque->qu_njstate[pjob->ji_qs.ji_state]++; + } - /* increment this user's job count for this queue */ - if (LOGLEVEL >= 6) - { - snprintf(log_buf, sizeof(log_buf), "jobs queued job id %s for %s", pjob->ji_qs.ji_jobid, pque->qu_qs.qu_name); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - } - - increment_queued_jobs(pque->qu_uih, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); - increment_queued_jobs(&users, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + /* increment this user's job count for this queue */ + if (LOGLEVEL >= 6) + { + snprintf(log_buf, sizeof(log_buf), "jobs queued job id %s for %s", pjob->ji_qs.ji_jobid, pque->qu_qs.qu_name); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); } + increment_queued_jobs(pque->qu_uih, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + increment_queued_jobs(&users, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + if ((pjob->ji_is_array_template) || (pjob->ji_arraystructid[0] == '\0')) { @@ -1996,8 +1996,8 @@ int chk_svr_resc_limit( static int count_queued_jobs( - pbs_queue *pque, /* I */ - const char *user) /* I */ + pbs_queue *pque, /* I */ + const char *user) /* I */ { int num_jobs = 0; diff --git a/src/server/user_info.c b/src/server/user_info.c index 82cd332bd8..50fbe28cbb 100644 --- a/src/server/user_info.c +++ b/src/server/user_info.c @@ -150,7 +150,6 @@ unsigned int count_jobs_submitted( - int can_queue_new_job( char *user_name, @@ -181,7 +180,6 @@ int can_queue_new_job( - int increment_queued_jobs( user_info_holder *uih, @@ -193,7 +191,6 @@ int increment_queued_jobs( user_info *ui; unsigned bit = COUNTED_GLOBALLY; - /* If pbs_server is restarting we may get jobs in a completed state. we do not want to count these jobs as queued */ if (pjob->ji_qs.ji_state == JOB_STATE_COMPLETE) @@ -204,35 +201,40 @@ int increment_queued_jobs( // Prevent recounting a job if (pjob->ji_queue_counted & bit) - return(PBSE_NONE); + return(rc); + + // Only increment the count for array templates and normal jobs + if ((pjob->ji_is_array_template == TRUE) || + ((pjob->ji_wattr[JOB_ATR_job_array_id].at_flags & ATR_VFLAG_SET) == 0)) + { + unsigned int num_submitted = count_jobs_submitted(pjob); + std::string uname(user_name); - unsigned int num_submitted = count_jobs_submitted(pjob); - std::string uname(user_name); - - remove_server_suffix(uname); + remove_server_suffix(uname); - uih->lock(); + uih->lock(); - /* get the user if there is one */ - if ((ui = uih->find(uname)) != NULL) - { - ui->num_jobs_queued += num_submitted; - } - else - { - /* user doesn't exist, create a new one and insert */ - ui = (user_info *)calloc(1, sizeof(user_info)); - ui->user_name = strdup(uname.c_str()); - ui->num_jobs_queued = num_submitted; - - if (!uih->insert(ui,ui->user_name)) + /* get the user if there is one */ + if ((ui = uih->find(uname)) != NULL) { - rc = ENOMEM; - log_err(rc, __func__, "Can't resize the user info array"); + ui->num_jobs_queued += num_submitted; } - } + else + { + /* user doesn't exist, create a new one and insert */ + ui = (user_info *)calloc(1, sizeof(user_info)); + ui->user_name = strdup(uname.c_str()); + ui->num_jobs_queued = num_submitted; - uih->unlock(); + if (!uih->insert(ui, ui->user_name)) + { + rc = ENOMEM; + log_err(rc, __func__, "Can't resize the user info array"); + } + } + + uih->unlock(); + } // Mark this job as being counted as queued pjob->ji_queue_counted |= bit; diff --git a/src/test/user_info/test_uut.c b/src/test/user_info/test_uut.c index 9903135e9e..c4194160e0 100644 --- a/src/test/user_info/test_uut.c +++ b/src/test/user_info/test_uut.c @@ -146,12 +146,24 @@ START_TEST(increment_queued_jobs_test) // Enqueue the job without resetting to make sure the count doesn't change fail_unless(increment_queued_jobs(&users, strdup("tom@napali"), &pjob) == 0); fail_unless(get_num_queued(&users, "tom") == 3, "Shouldn't have incremented with the bit set"); + + // Make sure array subjobs are a no-op + pjob.ji_queue_counted = 0; + pjob.ji_is_array_template = FALSE; + pjob.ji_wattr[JOB_ATR_job_array_id].at_flags = ATR_VFLAG_SET; + fail_unless(increment_queued_jobs(&users, strdup("tom"), &pjob) == 0); + fail_unless(get_num_queued(&users, "tom") == 3, "Shouldn't have incremented for an array sub-job"); + fail_unless(pjob.ji_queue_counted != 0); + + pjob.ji_queue_counted = 0; + pjob.ji_wattr[JOB_ATR_job_array_id].at_flags = 0; + fail_unless(increment_queued_jobs(&users, strdup("tom"), &pjob) == 0); + fail_unless(get_num_queued(&users, "tom") == 4, "Should have incremented for non array sub-job"); } END_TEST - START_TEST(decrement_queued_jobs_test) { users.lock(); From 5f3d5f8fa7202a7efba407b3f1b5469cbaef2bb0 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 16 Feb 2017 17:14:00 -0700 Subject: [PATCH 152/312] TRQ-3913 - set proper default gpu mode based on CUDA library version --- src/server/node_manager.c | 11 +++++++++++ src/test/node_manager/test_uut.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index e67ffee97d..38d08bb4d4 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -198,6 +198,9 @@ int default_gpu_mode = -1; #ifndef MAX_BM #define MAX_BM 64 #endif +#ifdef NVML_API +#include +#endif int handle_complete_first_time(job *pjob); int is_compute_node(char *node_id); @@ -2166,7 +2169,15 @@ int proplist( if (deflt_gpu_mode != NULL) gpu_mode_rqstd = default_gpu_mode; else + { +#if defined(NVML_API_VERSION) && (NVML_API_VERSION >= 8) + // exclusive thread mode deprecated starting in version 8 + // so use exlusive process instead + gpu_mode_rqstd = gpu_exclusive_process; +#else gpu_mode_rqstd = gpu_exclusive_thread; +#endif + } } else { diff --git a/src/test/node_manager/test_uut.c b/src/test/node_manager/test_uut.c index a09eefe8b2..b18389a7ee 100644 --- a/src/test/node_manager/test_uut.c +++ b/src/test/node_manager/test_uut.c @@ -13,6 +13,10 @@ #include "server.h" #include "json/json.h" #include "complete_req.hpp" +#include "pbs_nodes.h" +#ifdef NVML_API +#include +#endif const char *exec_hosts = "napali/0+napali/1+napali/2+napali/50+napali/4+l11/0+l11/1+l11/2+l11/3"; char buf[4096]; @@ -44,6 +48,7 @@ int remove_job_from_nodes_mics(struct pbsnode *pnode, job *pjob); void update_failure_counts(const char *node_name, int rc); void check_node_jobs_existence(struct work_task *pwt); int add_job_to_gpu_subnode(pbsnode *pnode, gpusubn &gn, job *pjob); +int proplist(char**, struct prop**, int*, int*, int*); @@ -55,6 +60,7 @@ extern int decode_resc_count; extern bool conn_success; extern bool alloc_br_success; extern bool cray_enabled; +extern int gpu_mode_rqstd; START_TEST(free_nodes_test) @@ -946,6 +952,25 @@ START_TEST(place_subnodes_in_hostlist_job_exclusive_test) END_TEST +#ifdef NVML_API +START_TEST(test_proplist) + { + const char *str[] = {"gpus=1"}; + struct prop *plist= NULL; + int node_req; + int gpu_req; + int mic_req; + + fail_unless(proplist((char **)str, &plist, &node_req, &gpu_req, &mic_req) == PBSE_NONE); +#if defined(NVML_API_VERSION) && (NVML_API_VERSION >= 8) + fail_unless(gpu_mode_rqstd == gpu_exclusive_process); +#else + fail_unless(gpu_mode_rqstd == gpu_exclusive_thread); +#endif + } +END_TEST +#endif + Suite *node_manager_suite(void) { Suite *s = suite_create("node_manager_suite methods"); @@ -1008,6 +1033,12 @@ Suite *node_manager_suite(void) tcase_add_test(tc_core, check_node_jobs_exitence_test); suite_add_tcase(s, tc_core); +#ifdef NVML_API + tc_core = tcase_create("test_proplist"); + tcase_add_test(tc_core, test_proplist); + suite_add_tcase(s, tc_core); +#endif + return(s); } From 9c07aa3797b641786f630b01d5e7a5b739ab7465 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 16 Feb 2017 17:37:36 -0700 Subject: [PATCH 153/312] TRQ-3913 - fix unit test --- src/test/node_manager/test_uut.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/node_manager/test_uut.c b/src/test/node_manager/test_uut.c index b18389a7ee..d9a2441890 100644 --- a/src/test/node_manager/test_uut.c +++ b/src/test/node_manager/test_uut.c @@ -48,7 +48,7 @@ int remove_job_from_nodes_mics(struct pbsnode *pnode, job *pjob); void update_failure_counts(const char *node_name, int rc); void check_node_jobs_existence(struct work_task *pwt); int add_job_to_gpu_subnode(pbsnode *pnode, gpusubn &gn, job *pjob); -int proplist(char**, struct prop**, int*, int*, int*); +int proplist(char **str, std::vector &plist, int *node_req, int *gpu_req, int *mic_req); @@ -956,12 +956,12 @@ END_TEST START_TEST(test_proplist) { const char *str[] = {"gpus=1"}; - struct prop *plist= NULL; + std::vector plist; int node_req; int gpu_req; int mic_req; - fail_unless(proplist((char **)str, &plist, &node_req, &gpu_req, &mic_req) == PBSE_NONE); + fail_unless(proplist((char **)str, plist, &node_req, &gpu_req, &mic_req) == PBSE_NONE); #if defined(NVML_API_VERSION) && (NVML_API_VERSION >= 8) fail_unless(gpu_mode_rqstd == gpu_exclusive_process); #else From b8867997c6f7276f90db5dac6da5ab54f511cde2 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 17 Feb 2017 16:07:51 -0700 Subject: [PATCH 154/312] Fix some unit test issues. --- .../receive_mom_communication/Makefile.am | 7 +++-- .../receive_mom_communication/scaffolding.c | 26 +++++++++++++++++++ src/test/req_delete/test_uut.c | 1 + 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/test/receive_mom_communication/Makefile.am b/src/test/receive_mom_communication/Makefile.am index 1584d00804..7794457d47 100644 --- a/src/test/receive_mom_communication/Makefile.am +++ b/src/test/receive_mom_communication/Makefile.am @@ -2,5 +2,8 @@ include ../Makefile_Server.ut libuut_la_SOURCES = ${PROG_ROOT}/process_mom_update.c ${PROG_ROOT}/receive_mom_communication.c \ - ${PROG_ROOT}/../lib/Libutils/machine.cpp ${PROG_ROOT}/../lib/Libutils/numa_pci_device.cpp \ - ${PROG_ROOT}/../lib/Libutils/numa_socket.cpp ${PROG_ROOT}/../lib/Libutils/numa_chip.cpp + ${PROG_ROOT}/../lib/Libutils/machine.cpp \ + ${PROG_ROOT}/../lib/Libutils/numa_pci_device.cpp \ + ${PROG_ROOT}/../lib/Libutils/numa_socket.cpp \ + ${PROG_ROOT}/../lib/Libutils/numa_chip.cpp \ + ${PROG_ROOT}/../lib/Libutils/jsoncpp.cpp diff --git a/src/test/receive_mom_communication/scaffolding.c b/src/test/receive_mom_communication/scaffolding.c index 0dcd691b81..8d7caa21ca 100644 --- a/src/test/receive_mom_communication/scaffolding.c +++ b/src/test/receive_mom_communication/scaffolding.c @@ -934,3 +934,29 @@ pbsnode *authorized_hosts::get_authorized_node(unsigned long addr, unsigned shor authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; + +const char *LPROCS = "lprocs"; +const char *NODE = "node"; +const char *NODES = "nodes"; +const char *SOCKET = "socket"; +const char *SOCKETS = "sockets"; +const char *NUMA_NODE = "numanode"; +const char *NUMA_NODES = "numanodes"; +const char *CORE = "core"; +const char *CORES = "cores"; +const char *THREAD = "thread"; +const char *THREADS = "threads"; +const char *MEMORY = "memory"; +const char *SWAP = "swap"; +const char *DISK = "disk"; + +const char *ALLOCATION = "allocation"; +const char *ALLOCATIONS = "allocations"; +const char *MEM = "mem"; +const char *GPUS = "gpus"; +const char *MICS = "mics"; +const char *CORES_ONLY = "cores_only"; +const char *JOBID = "jobid"; +const char *CPUS = "cpus"; +const char *EXCLUSIVE = "exclusive"; +const char *OS_INDEX = "os_index"; diff --git a/src/test/req_delete/test_uut.c b/src/test/req_delete/test_uut.c index bacbb569a4..4dd24e3df1 100644 --- a/src/test/req_delete/test_uut.c +++ b/src/test/req_delete/test_uut.c @@ -292,6 +292,7 @@ START_TEST(test_force_purge_work) { job *pjob = new job(); + sprintf(pjob->ji_qs.ji_jobid, "17.roshar"); pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str = strdup("bob"); depend_term_called = 0; force_purge_work(pjob); From 18246573cbfdb4a71659cd06e57098c95f4d723a Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 17 Feb 2017 16:03:17 -0700 Subject: [PATCH 155/312] TRQ-3934. Fix some translation of memory requirements from -l to -L. Conflicts: src/lib/Libattr/complete_req.cpp src/test/complete_req/test_uut.c --- src/lib/Libattr/complete_req.cpp | 110 ++++++++++++++++--------------- src/lib/Libattr/req.cpp | 5 +- src/test/complete_req/test_uut.c | 9 ++- src/test/req/test_uut.c | 14 +++- 4 files changed, 77 insertions(+), 61 deletions(-) diff --git a/src/lib/Libattr/complete_req.cpp b/src/lib/Libattr/complete_req.cpp index 039cb29740..ccacfdb15f 100644 --- a/src/lib/Libattr/complete_req.cpp +++ b/src/lib/Libattr/complete_req.cpp @@ -73,13 +73,12 @@ complete_req::complete_req( int task_count = 0; int execution_slots = 0; unsigned long long mem_values[4]; - int active_index[2]; + bool using_pmem = false; + bool using_pvmem = false; if (resources == NULL) return; - active_index[0] = _MEM_; - active_index[1] = _VMEM_; memset(mem_values, 0, sizeof(mem_values)); for (size_t i = 0; i < resources->size(); i++) @@ -144,25 +143,47 @@ complete_req::complete_req( } } - // Set mem and swap from mem_values - unsigned long long mem = mem_values[_MEM_]; - if ((legacy_vmem == false) && - (task_count != 0)) - mem /= task_count; - if (mem_values[_PMEM_] > mem) + // Set mem_per_task and swap_per_task from mem_values + unsigned long long mem_per_task = mem_values[_MEM_]; + if (task_count != 0) + mem_per_task /= task_count; + + if (this->reqs.size() > 0) { - active_index[0] = _PMEM_; - mem = mem_values[_PMEM_]; + // Nodes request - check if pmem * ppn > mem_per_task + req &r = this->reqs[0]; + + if (mem_values[_PMEM_] * r.getExecutionSlots() > mem_per_task) + { + mem_per_task = mem_values[_PMEM_]; + using_pmem = true; + } } + else if (mem_values[_PMEM_] > mem_per_task) + { + mem_per_task = mem_values[_PMEM_]; + using_pmem = true; + } + + unsigned long long vmem_per_task = mem_values[_VMEM_]; + if (task_count != 0) + vmem_per_task /= task_count; - unsigned long long vmem = mem_values[_VMEM_]; - if ((legacy_vmem == false) && - (task_count != 0)) - vmem /= task_count; - if (mem_values[_PVMEM_] > vmem) + if (this->reqs.size() > 0) { - active_index[1] = _PVMEM_; - vmem = mem_values[_PVMEM_]; + // Nodes request - check if pvmem * ppn > vmem_per_task + req &r = this->reqs[0]; + + if (mem_values[_PVMEM_] * r.getExecutionSlots() > vmem_per_task) + { + vmem_per_task = mem_values[_PVMEM_]; + using_pvmem = true; + } + } + else if (mem_values[_PVMEM_] > vmem_per_task) + { + vmem_per_task = mem_values[_PVMEM_]; + using_pvmem = true; } if (this->reqs.size() == 0) @@ -176,8 +197,8 @@ complete_req::complete_req( r.set_placement_type(place_legacy); } - r.set_memory(mem); - r.set_swap(vmem); + r.set_memory(mem_per_task * task_count); + r.set_swap(vmem_per_task * task_count); if (execution_slots != 0) r.set_execution_slots(execution_slots); @@ -187,48 +208,29 @@ complete_req::complete_req( else { // Handle the case where a -lnodes request was made - if (mem != 0) + if (mem_per_task != 0) { - if (active_index[0] == _MEM_) - { - for (unsigned int i = 0; i < this->reqs.size(); i++) - { - req &r = this->reqs[i]; - r.set_memory(mem); - } - } - else if (active_index[0] == _PMEM_) + for (unsigned int i = 0; i < this->reqs.size(); i++) { - for (unsigned int i = 0; i < this->reqs.size(); i++) - { - req &r = this->reqs[i]; - int ppn_per_req = r.get_execution_slots(); - - r.set_memory(mem * ppn_per_req); - } + req &r = this->reqs[i]; + // When using pmem, multiply by ppn + if (using_pmem == true) + r.set_memory(mem_per_task * r.getTaskCount() * r.getExecutionSlots()); + else + r.set_memory(mem_per_task * r.getTaskCount()); } } - if (vmem != 0) + if (vmem_per_task != 0) { // Handle the case where a -lnodes request was made - if (active_index[1] == _VMEM_) - { - for (unsigned int i = 0; i < this->reqs.size(); i++) - { - req &r = this->reqs[i]; - r.set_swap(vmem); - } - } - else if (active_index[1] == _PVMEM_) + for (unsigned int i = 0; i < this->reqs.size(); i++) { - for (unsigned int i = 0; i < this->reqs.size(); i++) - { - req &r = this->reqs[i]; - int ppn_per_req = r.get_execution_slots(); - - r.set_swap(vmem * ppn_per_req); - } + req &r = this->reqs[i]; + if (using_pvmem == true) + r.set_swap(vmem_per_task * r.getTaskCount() * r.getExecutionSlots()); + else + r.set_swap(vmem_per_task * r.getTaskCount()); } } } diff --git a/src/lib/Libattr/req.cpp b/src/lib/Libattr/req.cpp index c8d78262d0..6380d3c4bf 100644 --- a/src/lib/Libattr/req.cpp +++ b/src/lib/Libattr/req.cpp @@ -2413,7 +2413,6 @@ int req::get_num_tasks_for_host( } task_count = num_ppn / this->execution_slots; - } } } @@ -2529,8 +2528,8 @@ unsigned long long req::get_memory_for_host( const std::string &host) const { - int num_tasks = this->get_num_tasks_for_host(host); - unsigned long long mem = this->mem * num_tasks; + int num_tasks = this->get_num_tasks_for_host(host); + unsigned long long mem = (this->mem * num_tasks) / this->task_count; return(mem); } // END get_memory_for_host() diff --git a/src/test/complete_req/test_uut.c b/src/test/complete_req/test_uut.c index d30b0780f6..8a08b169e6 100644 --- a/src/test/complete_req/test_uut.c +++ b/src/test/complete_req/test_uut.c @@ -164,6 +164,7 @@ START_TEST(test_constructor) fail_unless(list4.req_count() == 1); const req &rl2 = list4.get_req(0); fail_unless(rl2.getTaskCount() == 1); + // pmem should get multiplied because nodes=1:ppn=2 fail_unless(rl2.getMemory() == 160, "pmem is %lu", rl2.getMemory()); resources.clear(); @@ -174,6 +175,7 @@ START_TEST(test_constructor) const req &rl3 = list5.get_req(0); fail_unless(rl3.getTaskCount() == 1); fail_unless(rl3.getMemory() == 0, "pvmem is %lu", rl3.getMemory()); + // pvmem should get multiplied because nodes=1:ppn=2 is one task fail_unless(rl3.getSwap() == 160, "pvmem is %lu", rl3.getSwap()); resources.clear(); @@ -183,8 +185,8 @@ START_TEST(test_constructor) fail_unless(list6.req_count() == 1); const req &rl4 = list6.get_req(0); fail_unless(rl4.getTaskCount() == 1); - fail_unless(rl4.getMemory() == 0, "pvmem is %lu", rl4.getMemory()); - fail_unless(rl4.getSwap() == 80, "pvmem is %lu", rl4.getSwap()); + fail_unless(rl4.getMemory() == 0, "vmem is %lu", rl4.getMemory()); + fail_unless(rl4.getSwap() == 80, "vmem is %lu", rl4.getSwap()); resources.clear(); add_resource(resources, "procs", NULL, 2, -1); @@ -193,7 +195,8 @@ START_TEST(test_constructor) fail_unless(list7.req_count() == 1); const req &rl7 = list7.get_req(0); fail_unless(rl7.getTaskCount() == 2, "task count is %d", rl7.getTaskCount()); - fail_unless(rl7.getMemory() == 1024, "mem = %lu", rl7.getMemory()); + // 1024 * 2 = 2048, multiply by 2 because procs=2 + fail_unless(rl7.getMemory() == 2048, "mem = %lu", rl7.getMemory()); // Make sure that we'll set memory to the higher of pmem and mem, and set swap // as well for the same job diff --git a/src/test/req/test_uut.c b/src/test/req/test_uut.c index 446e0c02b5..47cdeaa60e 100644 --- a/src/test/req/test_uut.c +++ b/src/test/req/test_uut.c @@ -577,6 +577,7 @@ END_TEST START_TEST(test_get_memory_for_host) { req r; + req r2; std::string host = "napali"; unsigned long mem; @@ -587,9 +588,20 @@ START_TEST(test_get_memory_for_host) r.set_value("task_count", "5", false); r.set_value("thread_usage_policy", "use threads", false); r.set_value("hostlist", "napali:ppn=32", false); + + r2.set_value("index", "0", false); + r2.set_value("lprocs", "all", false); + r2.set_value("memory", "1024kb", false); + r2.set_value("swap", "1024kb", false); + r2.set_value("task_count", "1", false); + r2.set_value("thread_usage_policy", "use threads", false); + r2.set_value("hostlist", "napali:ppn=32", false); mem = r.get_memory_for_host(host); - fail_unless(mem != 0); + fail_unless(mem == 204); // 1024kb / 5 tasks = 204kb for this node + + mem = r2.get_memory_for_host(host); + fail_unless(mem == 1024); // 1024kb / 1 task = 1024kb for this node host = "right said fred"; mem = r.get_memory_for_host(host); From 98cddaf72543232a2e7da22d46b33890e50a808c Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 17 Feb 2017 17:08:36 -0700 Subject: [PATCH 156/312] TRQ-2978. Make the logging more specific for group failures. Conflicts: src/resmom/start_exec.c --- src/include/mom_func.h | 2 +- src/include/pbs_error_db.h | 1 + src/resmom/catch_child.c | 5 ++-- src/resmom/mom_req_quejob.c | 4 +--- src/resmom/start_exec.c | 33 ++++++++++++++------------- src/resmom/start_exec.h | 2 -- src/test/catch_child/scaffolding.c | 4 ++-- src/test/mom_comm/scaffolding.c | 11 +++++---- src/test/mom_req_quejob/scaffolding.c | 5 ++-- src/test/start_exec/test_uut.c | 12 +++++----- 10 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/include/mom_func.h b/src/include/mom_func.h index 6b08c0ea89..069f70198f 100644 --- a/src/include/mom_func.h +++ b/src/include/mom_func.h @@ -257,7 +257,7 @@ extern proc_stat_t *get_proc_stat(int pid); extern void term_job(job *); int TTmpDirName(job *, char *, int); -extern bool check_pwd(job *); +extern int check_pwd(job *); extern int task_save(task *) ; extern void DIS_rpp_reset(void); extern void checkret(char **, long); diff --git a/src/include/pbs_error_db.h b/src/include/pbs_error_db.h index 3720449f7e..919b7f70e1 100644 --- a/src/include/pbs_error_db.h +++ b/src/include/pbs_error_db.h @@ -299,6 +299,7 @@ PbsErrClient(PBSE_EOF, (char *)"This stream has already been closed. End of File PbsErrClient(PBSE_GPU_PROHIBITED_MODE, (char *)"Invalid gpu mode requested. Prohibited mode is not allowed. Check the spelling of the mode request for errors") PbsErrClient(PBSE_NODE_DELETED, (char *)"Node was deleted during work") PbsErrClient(PBSE_STATE_SLOT_LIMIT, (char *)"The requested state or substate can't be set from the job's current state (slot limit in effect).") +PbsErrClient(PBSE_BAD_GROUP, (char *)"Bad group entry") /* pbs client errors ceiling (max_client_err + 1) */ PbsErrClient(PBSE_CEILING, (char*)0) #endif diff --git a/src/resmom/catch_child.c b/src/resmom/catch_child.c index 3b13189454..53ff707b7f 100644 --- a/src/resmom/catch_child.c +++ b/src/resmom/catch_child.c @@ -1754,10 +1754,9 @@ void init_abort_jobs( DBPRT(("init_abort_jobs: setting grpcache for job %s\n", pj->ji_qs.ji_jobid)); - bool good; + - good = check_pwd(pj); - if (good == false) + if (check_pwd(pj) != PBSE_NONE) { /* somehow a job that was legally executing (had a password entry) * no longer has a password entry?? */ diff --git a/src/resmom/mom_req_quejob.c b/src/resmom/mom_req_quejob.c index 3eaf880b05..2211f52d55 100644 --- a/src/resmom/mom_req_quejob.c +++ b/src/resmom/mom_req_quejob.c @@ -754,10 +754,8 @@ void req_mvjobfile( return; } - bool good; - good = check_pwd(pj); if ((pj->ji_grpcache == NULL) && - (good == false)) + (check_pwd(pj) != PBSE_NONE)) { req_reject(PBSE_UNKJOBID, 0, preq, NULL, NULL); diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index 1721dfb1f0..9ab6be842f 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -441,8 +441,8 @@ void no_hang( } /* END no_hang() */ - -bool check_pwd( +/* TODO: change -1 to actual PBSE error codes */ +int check_pwd( job *pjob) /* I (modified) */ @@ -464,7 +464,7 @@ bool check_pwd( sprintf(log_buffer, "no user specified for job"); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - return(false); + return(-1); } /* we will retry if needed just to cover temporary problems */ @@ -488,14 +488,14 @@ bool check_pwd( ptr); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - return(false); + return(-1); } #ifdef __CYGWIN__ if (IamUserByName(ptr) == 0) { free_pwnam(pwdp, pwd_buf); - return(false); + return(-1); #endif /* __CYGWIN__ */ if (pjob->ji_grpcache != NULL) @@ -504,7 +504,7 @@ bool check_pwd( /* group cache previously loaded and cached */ free_pwnam(pwdp, pwd_buf); - return(true); + return(PBSE_NONE); } pjob->ji_qs.ji_un_type = JOB_UNION_TYPE_MOM; @@ -520,7 +520,7 @@ bool check_pwd( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); free_pwnam(pwdp, pwd_buf); - return(false); + return(-1); } strcpy(pjob->ji_grpcache->gc_homedir, pwdp->pw_dir); @@ -570,7 +570,7 @@ bool check_pwd( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); free_pwnam(pwdp, pwd_buf); - return(false); + return(PBSE_BAD_GROUP); } } /* END if (grpp != NULL) */ } @@ -592,7 +592,7 @@ bool check_pwd( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); free_pwnam(pwdp, pwd_buf); - return(false); + return(-1); } /* perform site specific check on validatity of account */ @@ -604,16 +604,17 @@ bool check_pwd( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); free_pwnam(pwdp, pwd_buf); - return(false); + return(-1); } /* SUCCESS */ free_pwnam(pwdp, pwd_buf); - return(true); + return(PBSE_NONE); } /* END check_pwd() */ + /** * @see send_sisters() - child - send ABORT request to sisters * @see start_exec() - parent @@ -2277,7 +2278,7 @@ int TMomFinalizeJob1( * we do this now to save a few things in the job structure */ - if (check_pwd(pjob) == false) + if (check_pwd(pjob) != PBSE_NONE) { log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buffer); @@ -5465,10 +5466,10 @@ int setup_process_launch_pipes( (kid_write < 0)) { log_err(-1, __func__, "Couldn't set up pipes to monitor launching a process"); - + return(-1); } - + return(PBSE_NONE); } // END setup_process_launch_pipes() @@ -6151,7 +6152,7 @@ int start_process( * to spawn tasks (ji_grpcache). */ - if (!check_pwd(pjob)) + if (check_pwd(pjob) != PBSE_NONE) { log_err(-1, __func__, log_buffer); @@ -7305,7 +7306,7 @@ int start_exec( /* Step 3.0 Validate/Initialize Environment */ /* check creds early because we need the uid/gid for TMakeTmpDir() */ - if (check_pwd(pjob) == false) + if (check_pwd(pjob) != PBSE_NONE) { sprintf(log_buffer, "bad credentials: job id %s", pjob->ji_qs.ji_jobid); log_err(-1, __func__, log_buffer); diff --git a/src/resmom/start_exec.h b/src/resmom/start_exec.h index f8934372d3..a35950e1d1 100644 --- a/src/resmom/start_exec.h +++ b/src/resmom/start_exec.h @@ -21,8 +21,6 @@ enum csa_chk_cmd #define B_THRESHOLD 2048 #define EXTRA_VARIABLE_SPACE 5120 -bool check_pwd(job *pjob); - int open_demux(u_long addr, int port); int is_joined(job *pjob); diff --git a/src/test/catch_child/scaffolding.c b/src/test/catch_child/scaffolding.c index 4daa1ccaed..11ba520251 100644 --- a/src/test/catch_child/scaffolding.c +++ b/src/test/catch_child/scaffolding.c @@ -1310,9 +1310,9 @@ void mom_job_purge(job *pjob) return; } -bool check_pwd(job *pjob) +int check_pwd(job *pjob) { - return false; + return(-1); } unsigned long gettime(resource *pres) diff --git a/src/test/mom_comm/scaffolding.c b/src/test/mom_comm/scaffolding.c index 09cf622d7d..705b4a610f 100644 --- a/src/test/mom_comm/scaffolding.c +++ b/src/test/mom_comm/scaffolding.c @@ -114,14 +114,15 @@ int exec_job_on_ms(job *pjob) } struct passwd *check_pwd_return; -bool check_pwd(job *pjob) + +int check_pwd(job *pjob) { - bool good = false; - if(check_pwd_return == NULL) + int rc = -1; + if (check_pwd_return == NULL) { - good = true; + rc = PBSE_NONE; } - return(good); + return(rc); } int mom_do_poll(job *pjob) diff --git a/src/test/mom_req_quejob/scaffolding.c b/src/test/mom_req_quejob/scaffolding.c index 583f5a8813..900766a5d0 100644 --- a/src/test/mom_req_quejob/scaffolding.c +++ b/src/test/mom_req_quejob/scaffolding.c @@ -56,10 +56,9 @@ void mom_job_purge(job *pjob) exit(1); } -bool check_pwd(job *pjob) +int check_pwd(job *pjob) { - fprintf(stderr, "The call to check_pwd needs to be mocked!!\n"); - exit(1); + return(0); } int reply_send_mom(struct batch_request *request_mom) diff --git a/src/test/start_exec/test_uut.c b/src/test/start_exec/test_uut.c index d985c71eb8..203b948d68 100644 --- a/src/test/start_exec/test_uut.c +++ b/src/test/start_exec/test_uut.c @@ -529,34 +529,34 @@ END_TEST START_TEST(test_check_pwd_euser) { job *pjob = (job *)calloc(1, sizeof(job)); - bool pwd = false; + int pwd = -1; pwd = check_pwd(pjob); - fail_unless(pwd == false, "check_pwd succeeded with an empty job"); + fail_unless(pwd != PBSE_NONE, "check_pwd succeeded with an empty job"); bad_pwd = true; decode_str(&pjob->ji_wattr[JOB_ATR_euser], "euser", NULL, "rightsaidfred", 0); pwd = check_pwd(pjob); - fail_unless(pwd == false, "bad pwd fail"); + fail_unless(pwd != PBSE_NONE, "bad pwd fail"); bad_pwd = false; fail_init_groups = true; decode_str(&pjob->ji_wattr[JOB_ATR_euser], "euser", NULL, "dbeer", 0); pwd = check_pwd(pjob); - fail_unless(pwd == false, "bad grp fail"); + fail_unless(pwd != PBSE_NONE, "bad grp fail"); pjob->ji_grpcache = NULL; fail_init_groups = false; fail_site_grp_check = true; decode_str(&pjob->ji_wattr[JOB_ATR_euser], "euser", NULL, "dbeer", 0); pwd = check_pwd(pjob); - fail_unless(pwd == false, "bad site fail"); + fail_unless(pwd != PBSE_NONE, "bad site fail"); pjob->ji_grpcache = NULL; fail_site_grp_check = false; decode_str(&pjob->ji_wattr[JOB_ATR_euser], "euser", NULL, "dbeer", 0); pwd = check_pwd(pjob); - fail_unless(pwd == true); + fail_unless(pwd == PBSE_NONE); } END_TEST From e6ff18b1b5353ac3d4515a3c4b96485ecb916959 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 21 Feb 2017 14:15:38 -0700 Subject: [PATCH 157/312] TRQ-3941. Avoid a crash when arrays are submitted with a bad range. -t 0--1 --- src/lib/Libutils/u_misc.c | 8 ++++++++ src/test/u_misc/test_uut.c | 3 +++ 2 files changed, 11 insertions(+) diff --git a/src/lib/Libutils/u_misc.c b/src/lib/Libutils/u_misc.c index 2992db463b..6653f5e4b3 100644 --- a/src/lib/Libutils/u_misc.c +++ b/src/lib/Libutils/u_misc.c @@ -360,6 +360,14 @@ int translate_range_string_to_vector( if (*ptr == '-') { ptr++; + + if (!isdigit(*ptr)) + { + // Must be a digit at this point in time + rc = -1; + break; + } + curr = strtol(ptr, &ptr, 10); while (prev <= curr) diff --git a/src/test/u_misc/test_uut.c b/src/test/u_misc/test_uut.c index b1fec8bf9c..4366a76fde 100644 --- a/src/test/u_misc/test_uut.c +++ b/src/test/u_misc/test_uut.c @@ -113,6 +113,9 @@ START_TEST(test_translate_range_string_to_vector) fail_unless(indices.size() == 2); fail_unless(indices[0] == 6142); fail_unless(indices[1] == 6143); + + indices.clear(); + fail_unless(translate_range_string_to_vector("0--1", indices) != PBSE_NONE); } END_TEST From 3c1ba92a940bb5c9221bbc2d1dcbd6502efdc4dd Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 21 Feb 2017 15:22:21 -0700 Subject: [PATCH 158/312] Fix a unit test that fails if check is forking. --- src/test/machine/test_uut.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/machine/test_uut.c b/src/test/machine/test_uut.c index ce67b5fd99..ad7a1498ae 100644 --- a/src/test/machine/test_uut.c +++ b/src/test/machine/test_uut.c @@ -75,11 +75,13 @@ START_TEST(test_place_all_execution_slots) complete_req cr; pjob.ji_wattr[JOB_ATR_req_information].at_val.at_ptr = &cr; + my_req_count = 1; m.addSocket(2); sockets = 0; numa_node_count = 0; placed_all = 0; exec_slots = -1; + num_for_host = 1; m.place_job(&pjob, cpu, mem, "napali", false); fail_unless(placed_all == 2, "placed all: %d", placed_all); } @@ -97,6 +99,7 @@ START_TEST(test_spread_place) m.addSocket(2); + my_req_count = 1; sockets = 1; my_placement_type = 2; numa_node_count = 0; From f70b1b6f3cefef2acdccdfe043d5050099c1d59d Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 21 Feb 2017 16:17:17 -0700 Subject: [PATCH 159/312] TRQ-3933. Add some protections around the resource plugin code. --- src/server/job.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/server/job.cpp b/src/server/job.cpp index e56fba5388..da3372c989 100644 --- a/src/server/job.cpp +++ b/src/server/job.cpp @@ -170,7 +170,13 @@ void job::set_plugin_resource_usage_from_json( std::vectorkeys = resources.getMemberNames(); for (size_t i = 0; i < keys.size(); i++) - this->ji_plugin_usage_info[keys[i]] = resources[keys[i]].asString(); + { + if (keys[i] != "") + { + if (resources[keys[i]].empty() == false) + this->ji_plugin_usage_info[keys[i]] = resources[keys[i]].asString(); + } + } } From 3ceb697b5331a5c7e0e65349610855f5e3ef2f15 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 21 Feb 2017 17:04:50 -0700 Subject: [PATCH 160/312] TRQ-3927 - fix pbs_mom systemd unit file --- contrib/systemd/pbs_mom.service.in | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/contrib/systemd/pbs_mom.service.in b/contrib/systemd/pbs_mom.service.in index 5bb8cd32b2..108613c3e2 100644 --- a/contrib/systemd/pbs_mom.service.in +++ b/contrib/systemd/pbs_mom.service.in @@ -18,5 +18,16 @@ EnvironmentFile=-/etc/sysconfig/pbs_mom ExecStart=@sbindir@/pbs_mom -F -d $PBS_HOME $PBS_ARGS +# attempt graceful shutdown +# systemd will kill remaining processes if any +# exist after the following is executed +ExecStop=/bin/bash -c " \ + for i in {1..5}; do \ + kill -0 $MAINPID &>/dev/null || exit 0; \ + @sbindir@/momctl -s && exit; \ + sleep 1; \ + done \ +" + [Install] WantedBy=multi-user.target From 8b09b781f1c6e67c28b5ba80f0e1879cd7c7aab6 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 22 Feb 2017 09:48:20 -0700 Subject: [PATCH 161/312] TRQ-3940. Fix an infinite loop due to resetting loop conditions. --- src/resmom/mom_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index 53a9b540fe..d9d3d6bf11 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -3826,9 +3826,9 @@ int job_over_limit( total = (index == 0) ? gettime(useresc) : getsize(useresc); #ifndef NUMA_SUPPORT - for (i = 0;i < pjob->ji_numnodes - 1;i++) + for (int j = 0; j < pjob->ji_numnodes - 1; j++) { - noderes *nr = &pjob->ji_resources[i]; + noderes *nr = &pjob->ji_resources[j]; total += ((index == 0) ? nr->nr_cput : nr->nr_mem); } From 11f9e97802e47a9ed3a73015835283bd304ea70a Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 22 Feb 2017 11:50:00 -0700 Subject: [PATCH 162/312] TRQ-3927 - pbs_mom unit file kills process only; adjust DBPRT() macro --- contrib/systemd/pbs_mom.service.in | 4 ++-- src/include/log.h | 2 +- src/lib/Libnet/rm.c | 8 ++------ src/resmom/mom_main.c | 5 +++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/contrib/systemd/pbs_mom.service.in b/contrib/systemd/pbs_mom.service.in index 108613c3e2..09be264aab 100644 --- a/contrib/systemd/pbs_mom.service.in +++ b/contrib/systemd/pbs_mom.service.in @@ -18,9 +18,9 @@ EnvironmentFile=-/etc/sysconfig/pbs_mom ExecStart=@sbindir@/pbs_mom -F -d $PBS_HOME $PBS_ARGS +KillMode=process + # attempt graceful shutdown -# systemd will kill remaining processes if any -# exist after the following is executed ExecStop=/bin/bash -c " \ for i in {1..5}; do \ kill -0 $MAINPID &>/dev/null || exit 0; \ diff --git a/src/include/log.h b/src/include/log.h index 173a75e83c..0d90ce8516 100644 --- a/src/include/log.h +++ b/src/include/log.h @@ -94,7 +94,7 @@ /* ** Set up a debug print macro. */ -#ifdef NDEBUG +#if defined(NDEBUG) || (DEBUG == 0) #define DBPRT(x) #else #define DBPRT(x) printf x; diff --git a/src/lib/Libnet/rm.c b/src/lib/Libnet/rm.c index b59c4cd522..b34e78682a 100644 --- a/src/lib/Libnet/rm.c +++ b/src/lib/Libnet/rm.c @@ -1044,10 +1044,6 @@ extern "C" int activereq(void) { -#ifndef NDEBUG - static char id[] = "activereq"; -#endif - int i; int num; int PollArraySize; @@ -1063,7 +1059,7 @@ int activereq(void) if (PollArray == NULL) { // no memory - DBPRT(("%s: malloc %d %s\n", id, errno, pbs_strerror(errno))) + DBPRT(("%s: malloc %d %s\n", __func__, errno, pbs_strerror(errno))) return(-1); } @@ -1092,7 +1088,7 @@ int activereq(void) if (num == -1) { - DBPRT(("%s: poll %d %s\n", id, errno, pbs_strerror(errno))) + DBPRT(("%s: poll %d %s\n", __func__, errno, pbs_strerror(errno))) free(PollArray); return -1; } diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index d9d3d6bf11..f6b062f693 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -3334,13 +3334,14 @@ int do_tcp( default: { - struct sockaddr_in *addr = NULL; struct sockaddr s_addr; unsigned int len = sizeof(s_addr); if (getpeername(chan->sock, &s_addr, &len) == 0) { - addr = (struct sockaddr_in *)&s_addr; +#if DEBUG > 0 + struct sockaddr_in *addr = (struct sockaddr_in *)&s_addr; +#endif DBPRT(("%s: unknown request %d from %s", __func__, proto, netaddr(addr))) } From d0b33dbb6507fda5adbd183ec65e7dacbe6e3534 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 24 Feb 2017 11:59:42 -0700 Subject: [PATCH 163/312] TRQ-3878. Add new options for mail_body_fmt. Added %o (owner), %q (queue), %R (resources requested summary), %u (resources used summary), and %w (working directory). --- src/include/mail_throttler.hpp | 24 ++++++---- src/lib/Libattr/attr_str_conversion.c | 59 +++++++++++++++++++++++ src/server/mail_throttler.cpp | 62 +++++++++++++++++++++++- src/server/svr_format_job.c | 63 ++++++++++++++++++++++++- src/server/svr_mail.c | 43 +---------------- src/test/attr_str_conversion/test_uut.c | 45 ++++++++++++++++++ src/test/mail_throttler/scaffolding.c | 6 +++ src/test/mail_throttler/test_uut.c | 53 +++++++++++++++++---- src/test/svr_format_job/scaffolding.c | 58 ++++++++++++++++++++++- src/test/svr_format_job/test_uut.c | 57 ++++++++++++++++++++-- 10 files changed, 403 insertions(+), 67 deletions(-) diff --git a/src/include/mail_throttler.hpp b/src/include/mail_throttler.hpp index 3966ccb859..466e772bed 100644 --- a/src/include/mail_throttler.hpp +++ b/src/include/mail_throttler.hpp @@ -7,20 +7,28 @@ #include #include +#include "pbs_job.h" +#include "resource.h" class mail_info { public: - std::string mailto; - std::string exec_host; - std::string jobid; - std::string jobname; - std::string text; /* additional optional text */ - std::string errFile; - std::string outFile; - int mail_point; + std::string mailto; + std::string exec_host; + std::string jobid; + std::string jobname; + std::string text; // additional optional text + std::string errFile; + std::string outFile; + int mail_point; + std::string queue_name; + std::string owner; + std::string working_directory; + std::vector resources_requested; + std::vector resources_used; mail_info(); + mail_info(job *pjob); mail_info(const mail_info &other); mail_info &operator =(const mail_info &other); }; diff --git a/src/lib/Libattr/attr_str_conversion.c b/src/lib/Libattr/attr_str_conversion.c index 7a755b4f4b..b30a59b87c 100644 --- a/src/lib/Libattr/attr_str_conversion.c +++ b/src/lib/Libattr/attr_str_conversion.c @@ -177,6 +177,65 @@ int size_to_str( +/* + * resource_index_to_string() + * + * Writes the resource at index as a string: name=value + * + * @param output - the string output + * @param resources - the resource vector + * @param index - the index of the resource to print + * @return PBSE_NONE on success or -1 if index if bad + */ + +int resource_index_to_string( + + std::string &output, + std::vector &resources, + size_t index) + + { + if (index >= resources.size()) + return(-1); + + char buf[MAXLINE]; + int rc = PBSE_NONE; + + output = resources[index].rs_defin->rs_name; + output += "="; + + switch (resources[index].rs_value.at_type) + { + case ATR_TYPE_LONG: + + sprintf(buf, "%ld", resources[index].rs_value.at_val.at_long); + output += buf; + + break; + + case ATR_TYPE_STR: + + output += resources[index].rs_value.at_val.at_str; + + break; + + case ATR_TYPE_SIZE: + + size_to_str(resources[index].rs_value.at_val.at_size, buf, sizeof(buf)); + output += buf; + + break; + + default: + + rc = -1; + break; + + } + + return(rc); + } // END resource_index_to_string() + /* diff --git a/src/server/mail_throttler.cpp b/src/server/mail_throttler.cpp index 47f4aa787e..eae584e2c5 100644 --- a/src/server/mail_throttler.cpp +++ b/src/server/mail_throttler.cpp @@ -8,12 +8,67 @@ mail_info::mail_info() : mailto(), exec_host(), jobid(), jobname(), { } +// Constructor from job - update the default values with +mail_info::mail_info( + + job *pjob) : mailto(), exec_host(), jobid(), jobname(), text(), errFile(), outFile(), mail_point(0), + queue_name(), owner(), working_directory(), resources_requested(), resources_used() + + { + if (pjob != NULL) + { + if (pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str != NULL) + this->exec_host = pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str; + + this->jobid = pjob->ji_qs.ji_jobid; + + if (pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str != NULL) + this->jobname = pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str; + + // Initialize the output files + if (pjob->ji_wattr[JOB_ATR_join].at_flags & ATR_VFLAG_SET) + { + char *join_val = pjob->ji_wattr[JOB_ATR_join].at_val.at_str; + if (!strcmp(join_val, "oe")) + { + this->errFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; + this->outFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; + } + else if (!strcmp(join_val, "eo")) + { + this->errFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; + this->outFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; + } + } + + if (this->outFile.size() == 0) + this->outFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; + + if (this->errFile.size() == 0) + this->errFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; + + this->queue_name = pjob->ji_qs.ji_queue; + this->owner = pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str; + this->working_directory = pjob->ji_wattr[JOB_ATR_init_work_dir].at_val.at_str; + + if (pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr != NULL) + this->resources_requested = *((std::vector *)pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr); + + if (pjob->ji_wattr[JOB_ATR_resc_used].at_val.at_ptr != NULL) + this->resources_used = *((std::vector *)pjob->ji_wattr[JOB_ATR_resc_used].at_val.at_ptr); + } + } + // Copy constructor mail_info::mail_info( const mail_info &other) : mailto(other.mailto), exec_host(other.exec_host), jobid(other.jobid), jobname(other.jobname), text(other.text), errFile(other.errFile), - outFile(other.outFile), mail_point(other.mail_point) + outFile(other.outFile), mail_point(other.mail_point), + queue_name(other.queue_name), owner(other.owner), + working_directory(other.working_directory), + resources_requested(other.resources_requested), + resources_used(other.resources_used) { } @@ -31,6 +86,11 @@ mail_info &mail_info::operator =( this->errFile = other.errFile; this->outFile = other.outFile; this->mail_point = other.mail_point; + this->queue_name = other.queue_name; + this->owner = other.owner; + this->working_directory = other.working_directory; + this->resources_requested = other.resources_requested; + this->resources_used = other.resources_used; return(*this); } diff --git a/src/server/svr_format_job.c b/src/server/svr_format_job.c index 075cfff664..b78968d9b0 100644 --- a/src/server/svr_format_job.c +++ b/src/server/svr_format_job.c @@ -30,6 +30,7 @@ extern char *msg_job_stageinfail; extern char *msg_job_copychkptfail; extern char *msg_job_otherfail; +int resource_index_to_string(std::string &output, std::vector &resources, size_t index); void svr_format_job( @@ -143,7 +144,7 @@ void svr_format_job( } else if (*p == '%') /* format statement */ { - switch(p[1]) + switch (p[1]) { case 'd': /* details */ @@ -210,12 +211,72 @@ void svr_format_job( break; + case 'o': // owner + + fprintf(fh, "%s", mi->owner.c_str()); + p += 2; + + break; + + case 'q': // queue + + fprintf(fh, "%s", mi->queue_name.c_str()); + + p += 2; + + break; + + case 'R': // Resources requested + + { + std::string resource_str; + + if (mi->resources_requested.size() > 0) + fprintf(fh, "Resources Requested Summary\n"); + + for (size_t i = 0; i < mi->resources_requested.size(); i++) + { + resource_index_to_string(resource_str, mi->resources_requested, i); + fprintf(fh, "\t%s\n", resource_str.c_str()); + } + p += 2; + + break; + } + case 'r': /* reason */ if (reason != NULL) { fprintf(fh, "%s", reason); } + + p += 2; + + break; + + case 'u': // resources used summary + + { + std::string resource_str; + + if (mi->resources_used.size() > 0) + fprintf(fh, "Resources Used Summary\n"); + + for (size_t i = 0; i < mi->resources_used.size(); i++) + { + resource_index_to_string(resource_str, mi->resources_used, i); + fprintf(fh, "\t%s\n", resource_str.c_str()); + } + + p += 2; + + break; + } + + case 'w': // working directory + + fprintf(fh, "%s", mi->working_directory.c_str()); p += 2; break; diff --git a/src/server/svr_mail.c b/src/server/svr_mail.c index 5dd8967ba2..d7ca7f2500 100644 --- a/src/server/svr_mail.c +++ b/src/server/svr_mail.c @@ -438,36 +438,6 @@ void *send_the_mail( return(NULL); } /* END send_the_mail() */ - - - -void set_output_files( - - job *pjob, - mail_info *mi) - - { - if (pjob->ji_wattr[JOB_ATR_join].at_flags & ATR_VFLAG_SET) - { - char *join_val = pjob->ji_wattr[JOB_ATR_join].at_val.at_str; - if (!strcmp(join_val, "oe")) - { - mi->errFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; - mi->outFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; - } - else if (!strcmp(join_val, "eo")) - { - mi->errFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; - mi->outFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; - } - } - - if (mi->outFile.size() == 0) - mi->outFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; - - if (mi->errFile.size() == 0) - mi->errFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; - } // END set_output_files() @@ -590,7 +560,7 @@ void svr_mailowner( char mailto[1024]; char *domain = NULL; int i; - mail_info mi; + mail_info mi(pjob); bool no_force = false; struct array_strings *pas; @@ -784,17 +754,6 @@ void svr_mailowner( mi.mailto = mailto; mi.mail_point = mailpoint; - if (pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str != NULL) - mi.exec_host = pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str; - - mi.jobid = pjob->ji_qs.ji_jobid; - - if (pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str != NULL) - mi.jobname = pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str; - - if (mailpoint == (int) MAIL_END || mailpoint == MAIL_ABORT) - set_output_files(pjob, &mi); - if (text) { mi.text = text; diff --git a/src/test/attr_str_conversion/test_uut.c b/src/test/attr_str_conversion/test_uut.c index d68e36a91b..4fc6e0266a 100644 --- a/src/test/attr_str_conversion/test_uut.c +++ b/src/test/attr_str_conversion/test_uut.c @@ -6,11 +6,55 @@ #include #include "attribute.h" +#include "resource.h" #include "server.h" void appendEscapedXML(const char *xml, std::string &str); int size_to_str(struct size_value *szv, char *out, int space); int attr_to_str(std::string &output, attribute_def *at_def, pbs_attribute attr, bool xml); +int resource_index_to_string(std::string &output, std::vector &resources, size_t index); + + +START_TEST(test_resource_index_to_string) + { + std::string output; + resource_def ad[4]; + resource r[4]; + + for (int i = 0; i < 4; i++) + r[i].rs_defin = ad + i; + + ad[0].rs_name = "cput"; + ad[1].rs_name = "vmem"; + ad[2].rs_name = "mem"; + ad[3].rs_name = "stringy"; + + r[0].rs_value.at_type = ATR_TYPE_LONG; + r[0].rs_value.at_val.at_long = 200; + r[1].rs_value.at_type = ATR_TYPE_SIZE; + r[1].rs_value.at_val.at_size.atsv_num = 2048; + r[1].rs_value.at_val.at_size.atsv_shift = 10; + r[2].rs_value.at_type = ATR_TYPE_SIZE; + r[2].rs_value.at_val.at_size.atsv_num = 3050; + r[2].rs_value.at_val.at_size.atsv_shift = 10; + r[3].rs_value.at_type = ATR_TYPE_STR; + r[3].rs_value.at_val.at_str = strdup("bob"); + + std::vector resources; + for (int i = 0; i < 4; i++) + resources.push_back(r[i]); + + fail_unless(resource_index_to_string(output, resources, 0) == PBSE_NONE); + fail_unless(output == "cput=200"); + fail_unless(resource_index_to_string(output, resources, 1) == PBSE_NONE); + fail_unless(output == "vmem=2048kb"); + fail_unless(resource_index_to_string(output, resources, 2) == PBSE_NONE); + fail_unless(output == "mem=3050kb"); + fail_unless(resource_index_to_string(output, resources, 3) == PBSE_NONE); + fail_unless(output == "stringy=bob"); + fail_unless(resource_index_to_string(output, resources, 4) != PBSE_NONE); + } +END_TEST START_TEST(test_attr_to_str) @@ -107,6 +151,7 @@ Suite *attr_str_conversion_suite(void) Suite *s = suite_create("attr_str_conversion test suite methods"); TCase *tc_core = tcase_create("test_appendEscapedXML"); tcase_add_test(tc_core, test_appendEscapedXML); + tcase_add_test(tc_core, test_resource_index_to_string); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_size_to_str"); diff --git a/src/test/mail_throttler/scaffolding.c b/src/test/mail_throttler/scaffolding.c index 4de0cb0936..e7b3961f9f 100644 --- a/src/test/mail_throttler/scaffolding.c +++ b/src/test/mail_throttler/scaffolding.c @@ -1,4 +1,10 @@ #include +#include "pbs_job.h" + int pthread_mutex_lock(pthread_mutex_t *mutex) throw() { return(0); } + +job::job() {} +job::~job() {} + diff --git a/src/test/mail_throttler/test_uut.c b/src/test/mail_throttler/test_uut.c index 3761a4e8e0..02ad899625 100644 --- a/src/test/mail_throttler/test_uut.c +++ b/src/test/mail_throttler/test_uut.c @@ -1,11 +1,52 @@ #include #include -#include #include "mail_throttler.hpp" #include "pbs_error.h" +#include "resource.h" + +#include +START_TEST(test_job_constructor) + { + job pjob; + const char *jname = "highstorm_predictor"; + const char *owner = "dbeer@roshar"; + const char *queue = "batch"; + const char *jid = "1.nalthis"; + const char *hlist = "scadrial:ppn=16"; + const char *join = "oe"; + const char *outpath = "/home/dbeer/highstorms/"; + + std::vector resources; + + pjob.ji_wattr[JOB_ATR_jobname].at_val.at_str = strdup(jname); + pjob.ji_wattr[JOB_ATR_job_owner].at_val.at_str = strdup(owner); + pjob.ji_wattr[JOB_ATR_exec_host].at_val.at_str = strdup(hlist); + pjob.ji_wattr[JOB_ATR_join].at_flags = ATR_VFLAG_SET; + pjob.ji_wattr[JOB_ATR_join].at_val.at_str = strdup(join); + pjob.ji_wattr[JOB_ATR_outpath].at_val.at_str = strdup(outpath); + pjob.ji_wattr[JOB_ATR_init_work_dir].at_val.at_str = strdup(outpath); + pjob.ji_wattr[JOB_ATR_resource].at_val.at_ptr = &resources; + pjob.ji_wattr[JOB_ATR_resc_used].at_val.at_ptr = &resources; + + strcpy(pjob.ji_qs.ji_queue, queue); + strcpy(pjob.ji_qs.ji_jobid, jid); + + mail_info mi(&pjob); + + fail_unless(mi.errFile == outpath); + fail_unless(mi.outFile == outpath); + fail_unless(mi.jobid == jid); + fail_unless(mi.exec_host == hlist); + fail_unless(mi.jobname == jname); + fail_unless(mi.queue_name == queue); + fail_unless(mi.owner == owner); + fail_unless(mi.working_directory == outpath); + } +END_TEST + START_TEST(test_adding) { @@ -52,12 +93,6 @@ START_TEST(test_adding) END_TEST -START_TEST(test_two) - { - } -END_TEST - - Suite *mail_throttler_suite(void) { Suite *s = suite_create("mail_throttler test suite methods"); @@ -65,8 +100,8 @@ Suite *mail_throttler_suite(void) tcase_add_test(tc_core, test_adding); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_job_constructor"); + tcase_add_test(tc_core, test_job_constructor); suite_add_tcase(s, tc_core); return(s); diff --git a/src/test/svr_format_job/scaffolding.c b/src/test/svr_format_job/scaffolding.c index 684206f8c6..46bedc8f0f 100644 --- a/src/test/svr_format_job/scaffolding.c +++ b/src/test/svr_format_job/scaffolding.c @@ -2,6 +2,12 @@ #include #include /* fprintf */ +#include +#include + +#include "resource.h" +#include "mail_throttler.hpp" + const char *msg_job_stageinfail = "File stage in failed, see below.\nJob will be retried later, please investigate and correct problem."; const char *msg_job_start = "Begun execution"; const char *msg_job_end = "Execution terminated"; @@ -14,8 +20,56 @@ extern "C" { char *pbse_to_txt(int err) { - fprintf(stderr, "The call to pbse_to_txt to be mocked!!\n"); - exit(1); + return(strdup("Error")); } } +int resource_index_to_string( + + std::string &output, + std::vector &resources, + size_t index) + + { + static int rcount = 0; + + switch (rcount) + { + case 0: + + output = "walltime=100"; + break; + + case 1: + + output = "nodes=2:ppn=8"; + break; + + case 2: + + output="cput=800"; + break; + + case 3: + + output="vmem=8024mb"; + break; + + case 4: + + output="mem=7000mb"; + break; + + default: + break; + } + + rcount++; + + return(0); + } + +mail_info::mail_info() + { + } + diff --git a/src/test/svr_format_job/test_uut.c b/src/test/svr_format_job/test_uut.c index 8f67fda60a..bc8f15aabd 100644 --- a/src/test/svr_format_job/test_uut.c +++ b/src/test/svr_format_job/test_uut.c @@ -1,11 +1,60 @@ #include "license_pbs.h" /* See here for the software license */ -#include "test_svr_format_job.h" #include #include #include "pbs_error.h" -START_TEST(test_one) +#include "mail_throttler.hpp" +#include "test_svr_format_job.h" + +void svr_format_job(FILE *fh, mail_info *mi, const char *fmt); + +START_TEST(test_new_formats) { + const char *path = "./new_formats.txt"; + const char *format = "My job is owned by %o in queue %q and it executed in working directory %w\n%R%u"; + const char *result = "My job is owned by dbeer@sel in queue batch and it executed in working \ +directory /home/dbeer\nResources Requested Summary\n\twalltime=100\n\ +\tnodes=2:ppn=8\nResources Used Summary\n\tcput=800\n\tvmem=8024mb\n\ +\tmem=7000mb\n"; + mail_info mi; + mi.owner = "dbeer@sel"; + mi.queue_name = "batch"; + mi.working_directory = "/home/dbeer"; + + // These vectors have to be the correct size, but the actual information is put in through + // the scaffolding + std::vector requested; + std::vector used; + resource r; + for (int i = 0; i < 2; i++) + requested.push_back(r); + for (int i = 0; i < 3; i++) + used.push_back(r); + + mi.resources_requested = requested; + mi.resources_used = used; + + FILE *test_file = fopen(path, "w+"); + + if (test_file != NULL) + { + char buf[4096]; + svr_format_job(test_file, &mi, format); + fclose(test_file); + + test_file = fopen(path, "r"); + + if (test_file != NULL) + { + char *ptr = buf; + memset(ptr, 0, sizeof(buf)); + if (fread(ptr, 1, sizeof(buf) - 1, test_file) > 0) + { + fail_unless((!strcmp(buf, result)), "Expected:\n'%s'\nGot:\n'%s'", result, buf); + } + } + unlink(path); + } } END_TEST @@ -20,8 +69,8 @@ END_TEST Suite *svr_format_job_suite(void) { Suite *s = suite_create("svr_format_job_suite methods"); - TCase *tc_core = tcase_create("test_one"); - tcase_add_test(tc_core, test_one); + TCase *tc_core = tcase_create("test_new_formats"); + tcase_add_test(tc_core, test_new_formats); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_two"); From 79251f30fc9ede690e8bb199d4d243b81f8b2678 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 28 Feb 2017 09:26:56 -0700 Subject: [PATCH 164/312] TRQ-3753 - kill procs in cpuset before removing cgroup --- configure.ac | 1 + src/include/trq_cgroups.h | 1 + src/resmom/trq_cgroups.c | 69 +++++++++- src/test/Makefile.am | 4 + src/test/trq_cgroups/Makefile.am | 4 + src/test/trq_cgroups/scaffolding.c | 162 ++++++++++++++++++++++++ src/test/trq_cgroups/tasks | 1 + src/test/trq_cgroups/test_trq_cgroups.h | 9 ++ src/test/trq_cgroups/test_uut.c | 58 +++++++++ 9 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 src/test/trq_cgroups/Makefile.am create mode 100644 src/test/trq_cgroups/scaffolding.c create mode 100644 src/test/trq_cgroups/tasks create mode 100644 src/test/trq_cgroups/test_trq_cgroups.h create mode 100644 src/test/trq_cgroups/test_uut.c diff --git a/configure.ac b/configure.ac index 0dbbf1fe73..6e2a863f59 100644 --- a/configure.ac +++ b/configure.ac @@ -506,6 +506,7 @@ m4_ifdef([AM_SILENT_RULES],[ src/test/tmsock_recov/Makefile src/test/mom_mach/Makefile src/test/mom_start/Makefile + src/test/trq_cgroups/Makefile src/resmom/linux/test/Makefile src/resmom/linux/test/cpuset/Makefile src/resmom/linux/test/sys_file/Makefile diff --git a/src/include/trq_cgroups.h b/src/include/trq_cgroups.h index 389633fa4b..f765ef58e3 100644 --- a/src/include/trq_cgroups.h +++ b/src/include/trq_cgroups.h @@ -40,4 +40,5 @@ void trq_cg_delete_job_cgroups(const char *job_id, bool successfully_created); bool have_incompatible_dash_l_resource(pbs_attribute *pattr); int trq_cg_add_devices_to_cgroup(job *pjob); int init_torque_cgroups(); +void trq_cg_signal_tasks(const std::string& cgroup_path, int signal); #endif /* _TRQ_CGROUPS_H_ */ diff --git a/src/resmom/trq_cgroups.c b/src/resmom/trq_cgroups.c index ba50b06040..218b60c332 100644 --- a/src/resmom/trq_cgroups.c +++ b/src/resmom/trq_cgroups.c @@ -1880,6 +1880,69 @@ void trq_cg_delete_cgroup_path( } // END trq_cg_delete_cgroup_path() +/* + * trq_cg_signal_tasks() + * + * cgroup_path - the path to the cgroup whose tasks should be signaled + * signal - signal to send to each task in the cgroup + */ + +void trq_cg_signal_tasks( + + const string &cgroup_path, + int signal) + + { + std::string tasks_path(cgroup_path); + int npids; + int slept = 0; + FILE *fp; + char tid_str[1024]; + char log_buf[LOCAL_LOG_BUF_SIZE]; + struct stat statbuf; + + // build path to tasks file + tasks_path += "/tasks"; + + // do not continue if tasks file doesn't exist + if ((lstat(tasks_path.c_str(), &statbuf) != 0) || (!S_ISREG(statbuf.st_mode))) + return; + + do + { + npids = 0; + + // Signal each pid. If it takes more than 5 seconds to kill, give up. + + if ((fp = fopen(tasks_path.c_str(), "r")) != NULL) + { + while ((fgets(tid_str, sizeof(tid_str), fp)) != NULL) + { + int tid = atoi(tid_str); + + if (LOGLEVEL >= 9) + { + snprintf(log_buf, sizeof(log_buf), "sending signal %d to pid %d in cpuset %s", + signal, tid, cgroup_path.c_str()); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); + } + + kill(tid, signal); + npids++; + } + + fclose(fp); + } + + if ((signal == SIGKILL) && (npids)) + { + sleep(1); + slept++; + } + } while((signal == SIGKILL) && (npids > 0) && (slept <= 5)); + + } // END trq_cg_signal_tasks() + /* * trq_cg_delete_job_cgroups() @@ -1898,7 +1961,11 @@ void trq_cg_delete_job_cgroups( trq_cg_delete_cgroup_path(cg_cpu_path + job_id, successfully_created); trq_cg_delete_cgroup_path(cg_cpuacct_path + job_id, successfully_created); - + + // kill any procs in cpuset + trq_cg_signal_tasks(cg_cpuset_path + job_id, SIGKILL); + + // remove directory trq_cg_delete_cgroup_path(cg_cpuset_path + job_id, successfully_created); trq_cg_delete_cgroup_path(cg_memory_path + job_id, successfully_created); diff --git a/src/test/Makefile.am b/src/test/Makefile.am index 730fd5628c..f931459ac5 100644 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -80,6 +80,10 @@ if BUILDPMIX MOM_UT_DIRS += pmix_interface pmix_operation pmix_tracker endif +if BUILD_LINUX_CGROUPS + MOM_UT_DIRS += trq_cgroups +endif + if INCLUDE_PAM PAM_DIRS = pam endif diff --git a/src/test/trq_cgroups/Makefile.am b/src/test/trq_cgroups/Makefile.am new file mode 100644 index 0000000000..3d364a3e5a --- /dev/null +++ b/src/test/trq_cgroups/Makefile.am @@ -0,0 +1,4 @@ + +include ../Makefile_Mom.ut + +libuut_la_SOURCES = ${PROG_ROOT}/trq_cgroups.c diff --git a/src/test/trq_cgroups/scaffolding.c b/src/test/trq_cgroups/scaffolding.c new file mode 100644 index 0000000000..ff657f536e --- /dev/null +++ b/src/test/trq_cgroups/scaffolding.c @@ -0,0 +1,162 @@ +#include "license_pbs.h" /* See here for the software license */ +#include +#include /* fprintf */ +#include /* winsize */ +#include /* termios */ + +#include "pbs_ifl.h" /* PBS_MAXHOSTNAME */ +#include "log.h" /* LOG_BUF_SIZE */ +#include "pbs_job.h" /* job, task */ +#include "complete_req.hpp" +#include "machine.hpp" +#include "numa_node.hpp" + +bool global_trq_cg_signal_tasks_ok = false; + +char mom_alias[1]; + +char log_buffer[LOG_BUF_SIZE]; + +bool per_task; + +int is_login_node; + +int LOGLEVEL = 0; + +Machine::Machine() {} +Machine::~Machine() {} + +PCI_Device::~PCI_Device() {} + +Socket::~Socket() {} + +Chip::~Chip() {} + +Core::~Core() {} + +Machine this_node; + +req::req() {} + +bool req::is_per_task() const + + { + return(per_task); + } + +void req::get_task_host_name( + + std::string &task_host, + unsigned int task_index) + + {} + +int Machine::getTotalChips() const + { + return(0); + } + +int Machine::getTotalThreads() const + { + return(0); + } + +bool task_hosts_match(const char *one, const char *two) + { + return(true); + } + +allocation::allocation( + + const allocation &alloc) : cpu_indices(alloc.cpu_indices), memory(alloc.memory), cpus(alloc.cpus) + + { + this->jobid = alloc.jobid; + } + +allocation::allocation() : cpu_indices(), memory(0), cpus(0) + + { + this->jobid[0] = '\0'; + } + +allocation &allocation::operator =(const allocation &other) + { + return(*this); + } + +void log_err(int errnum, const char *routine, const char *text) {} + +int rmdir_ext(const char *dirname, int retry_limit) + { + return(0); + } + +void get_device_indices( + + const char *device_str, + std::vector &device_indices, + const char *suffix) + + {} + +int complete_req::req_count() const + { + return(0); + } + +bool have_incompatible_dash_l_resource( + + pbs_attribute *pattr) + + { + return(false); + } + +int req::get_task_allocation( + + unsigned int index, + allocation &task_allocation) const + + { + return(PBSE_NONE); + } + +int req::get_req_allocation_count() + + { + return(0); + } + +unsigned int complete_req::get_num_reqs() + { + return(1); + } + +req &complete_req::get_req(int i) + { + static req r; + + return(r); + } + +int req::getTaskCount() const + + { + return(0); + } + +void log_event( + + int eventtype, + int objclass, + const char *objname, + const char *text) + + { + if ((objname != NULL) && (strcmp(objname, "trq_cg_signal_tasks") == 0)) + { + if ((text != NULL) && (strcmp(text, "sending signal 15 to pid 9999 in cpuset .") == 0)) + global_trq_cg_signal_tasks_ok = true; + } + } diff --git a/src/test/trq_cgroups/tasks b/src/test/trq_cgroups/tasks new file mode 100644 index 0000000000..3c2df076c2 --- /dev/null +++ b/src/test/trq_cgroups/tasks @@ -0,0 +1 @@ +9999 diff --git a/src/test/trq_cgroups/test_trq_cgroups.h b/src/test/trq_cgroups/test_trq_cgroups.h new file mode 100644 index 0000000000..7fcfe357c9 --- /dev/null +++ b/src/test/trq_cgroups/test_trq_cgroups.h @@ -0,0 +1,9 @@ +#include "license_pbs.h" /* See here for the software license */ +#ifndef _TRQ_CGROUPS_CT_H +#define _TRQ_CGROUPS_CT_H +#include + +#define TRQ_CGROUPS_SUITE 1 +Suite *trq_cgroups_suite(); + +#endif /* _TRQ_CGROUPS_CT_H */ diff --git a/src/test/trq_cgroups/test_uut.c b/src/test/trq_cgroups/test_uut.c new file mode 100644 index 0000000000..13d85e206b --- /dev/null +++ b/src/test/trq_cgroups/test_uut.c @@ -0,0 +1,58 @@ +#include "license_pbs.h" /* See here for the software license */ +#include "mom_mach.h" +#include "test_trq_cgroups.h" +#include +#include +#include +#include + +#include "pbs_error.h" +#include "pbs_job.h" +#include "trq_cgroups.h" + +extern int LOGLEVEL; +extern bool global_trq_cg_signal_tasks_ok; + +START_TEST(test_trq_cg_signal_tasks) + { + LOGLEVEL = 9; + trq_cg_signal_tasks(".", SIGTERM); + fail_unless(global_trq_cg_signal_tasks_ok, "test_trq_cg_signal_tasks failed"); + } +END_TEST + +START_TEST(test_two) + { + } +END_TEST + +Suite *trq_cgroups_suite(void) + { + Suite *s = suite_create("trq_cgroups_suite methods"); + TCase *tc_core = tcase_create("test_trq_cg_signal_tasks"); + tcase_add_test(tc_core, test_trq_cg_signal_tasks); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_two"); + tcase_add_test(tc_core, test_two); + suite_add_tcase(s, tc_core); + + return s; + } + +void rundebug() + { + } + +int main(void) + { + int number_failed = 0; + SRunner *sr = NULL; + rundebug(); + sr = srunner_create(trq_cgroups_suite()); + srunner_set_log(sr, "trq_cgroups_suite.log"); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return number_failed; + } From 8571370bedbb7468eef77ae4359a18945fb12645 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 28 Feb 2017 09:44:17 -0700 Subject: [PATCH 165/312] TRQ-3882 - in stat_update() don't abort job if no longer running --- src/server/req_stat.c | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/server/req_stat.c b/src/server/req_stat.c index 1181feef3e..21a608aeee 100644 --- a/src/server/req_stat.c +++ b/src/server/req_stat.c @@ -924,18 +924,33 @@ void stat_update( directory is cleared, set its state to queued so job_abt doesn't think it is still running */ mutex_mgr job_mutex(pjob->ji_mutex, true); - - snprintf(log_buf, sizeof(log_buf), - "mother superior no longer recognizes %s as a valid job, aborting. Last reported time was %ld", - preq->rq_ind.rq_status.rq_id, pjob->ji_last_reported_time); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - - svr_setjobstate(pjob, JOB_STATE_QUEUED, JOB_SUBSTATE_ABORT, FALSE); - rel_resc(pjob); - job_mutex.set_unlock_on_exit(false); - job_abt(&pjob, "Job does not exist on node"); - /* TODO, if the job is rerunnable we should set its state back to queued */ + if (pjob->ji_qs.ji_state != JOB_STATE_RUNNING) + { + // don't abort if job no longer in running state + if (LOGLEVEL >= 6) + { + snprintf(log_buf, sizeof(log_buf), + "stat_update(): job %s is no longer in running state. Ignoring unknown job id reply from mom.", + preq->rq_ind.rq_status.rq_id); + + log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_JOB, __func__, log_buf); + } + } + else + { + snprintf(log_buf, sizeof(log_buf), + "mother superior no longer recognizes %s as a valid job, aborting. Last reported time was %ld", + preq->rq_ind.rq_status.rq_id, pjob->ji_last_reported_time); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); + + svr_setjobstate(pjob, JOB_STATE_QUEUED, JOB_SUBSTATE_ABORT, FALSE); + rel_resc(pjob); + job_mutex.set_unlock_on_exit(false); + job_abt(&pjob, "Job does not exist on node"); + + /* TODO, if the job is rerunnable we should set its state back to queued */ + } } } else From 87172c4ea30c6f326960a7cfc0f604cb24875837 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 28 Feb 2017 10:46:53 -0700 Subject: [PATCH 166/312] TRQ-3878. Fix a unit test problem. --- src/server/mail_throttler.cpp | 4 +++- src/test/svr_mail/scaffolding.c | 17 ++++++++++++++++- src/test/svr_mail/test_uut.c | 5 ++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/server/mail_throttler.cpp b/src/server/mail_throttler.cpp index eae584e2c5..b483b26500 100644 --- a/src/server/mail_throttler.cpp +++ b/src/server/mail_throttler.cpp @@ -49,7 +49,9 @@ mail_info::mail_info( this->queue_name = pjob->ji_qs.ji_queue; this->owner = pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str; - this->working_directory = pjob->ji_wattr[JOB_ATR_init_work_dir].at_val.at_str; + + if (pjob->ji_wattr[JOB_ATR_init_work_dir].at_flags & ATR_VFLAG_SET) + this->working_directory = pjob->ji_wattr[JOB_ATR_init_work_dir].at_val.at_str; if (pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr != NULL) this->resources_requested = *((std::vector *)pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr); diff --git a/src/test/svr_mail/scaffolding.c b/src/test/svr_mail/scaffolding.c index b9202e03ca..b6556fd5cd 100644 --- a/src/test/svr_mail/scaffolding.c +++ b/src/test/svr_mail/scaffolding.c @@ -128,7 +128,12 @@ void log_err(int errnum, const char *routine, const char *text) {} void log_record(int eventtype, int objclass, const char *objname, const char *text) {} void log_event(int eventtype, int objclass, const char *objname, const char *text) {} -job::job() {} +job::job() + { + memset(&this->ji_qs, 0, sizeof(this->ji_qs)); + memset(this->ji_wattr, 0, sizeof(this->ji_wattr)); + } + job::~job() {} bool empty_body = false; @@ -150,3 +155,13 @@ int get_svr_attr_str( return(0); } +int resource_index_to_string( + + std::string &output, + std::vector &resources, + size_t index) + + { + return(0); + } + diff --git a/src/test/svr_mail/test_uut.c b/src/test/svr_mail/test_uut.c index 7d2ff51e79..7997c4dba3 100644 --- a/src/test/svr_mail/test_uut.c +++ b/src/test/svr_mail/test_uut.c @@ -332,7 +332,10 @@ START_TEST(mail_point_p) job pjob; char p[]= "p"; - pjob.ji_wattr[JOB_ATR_mailpnts].at_val.at_str = p; + pjob.ji_wattr[JOB_ATR_mailpnts].at_val.at_str = p; + pjob.ji_wattr[JOB_ATR_outpath].at_val.at_str = strdup("/home/dbeer/forgery"); + pjob.ji_wattr[JOB_ATR_errpath].at_val.at_str = strdup("/home/dbeer/forgery"); + pjob.ji_wattr[JOB_ATR_job_owner].at_val.at_str = strdup("dbeer@sel"); svr_mailowner(&pjob, 1, 1, p); fail_unless((called == 0),"one"); From 14ae082931f8f2013ecf1053572f8f1e5d07878f Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 28 Feb 2017 10:49:34 -0700 Subject: [PATCH 167/312] TRQ-3714. Add the ability to set a specific memory and or swap value per mom. $max_physical_memory sets memory and $max_swap_memory sets swap. Conflicts: src/include/mom_config.h src/resmom/parse_config.c --- src/include/mom_config.h | 2 ++ src/resmom/linux/mom_mach.c | 54 +++++++++++++++++++++++--------- src/resmom/parse_config.c | 55 ++++++++++++++++++++++++++++----- src/test/mom_mach/scaffolding.c | 2 ++ src/test/mom_mach/test_uut.c | 51 ++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 22 deletions(-) diff --git a/src/include/mom_config.h b/src/include/mom_config.h index 5634cfd2c6..8198ec3f6b 100644 --- a/src/include/mom_config.h +++ b/src/include/mom_config.h @@ -174,6 +174,8 @@ extern int resend_join_job_wait_time; extern int mom_hierarchy_retry_time; extern int MOMJobDirStickySet; extern std::string presetup_prologue; +extern unsigned long max_memory; +extern unsigned long max_swap; struct specials { diff --git a/src/resmom/linux/mom_mach.c b/src/resmom/linux/mom_mach.c index 6e53b2cf6f..542f1d27e1 100644 --- a/src/resmom/linux/mom_mach.c +++ b/src/resmom/linux/mom_mach.c @@ -157,9 +157,9 @@ extern char path_meminfo[MAX_LINE]; ** local functions and data */ static const char *resi (struct rm_attribute *); -static const char *totmem (struct rm_attribute *); +const char *totmem (struct rm_attribute *); static const char *availmem (struct rm_attribute *); -static const char *physmem (struct rm_attribute *); +const char *physmem (struct rm_attribute *); static const char *ncpus (struct rm_attribute *); static const char *walltime (struct rm_attribute *); static const char *quota (struct rm_attribute *); @@ -4393,13 +4393,19 @@ const char *nusers( - +/* + * totmem - Determine the total amount of memory (RAM + Swap) on this host + * + * @return the total memory in kb as a string in the format: kb, or NULL on error + */ const char *totmem( struct rm_attribute *attrib) { - proc_mem_t *mm; + proc_mem_t *mm; + unsigned long total = 0; + unsigned long swap; if (attrib) { @@ -4428,16 +4434,32 @@ const char *totmem( log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer); } - sprintf(ret_string, "%lukb", + total = mm->mem_total >> 10; // Convert from bytes to kb - (ulong)((mm->mem_total >> 10) + (mm->swap_total >> 10))); /* KB */ + if ((max_memory > 0) && + (max_memory < total)) + total = max_memory; + + swap = mm->swap_total >> 10; // Convert from bytes to kb + + if ((max_swap > 0) && + (max_swap < swap)) + total += max_swap; + else + total += swap; + + sprintf(ret_string, "%lukb", total); return(ret_string); } /* END totmem() */ - +/* + * availmem() - Get the amount of available RAM + Swap for this host + * + * @return the available memory in kb as a string in the format: kb, or NULL on error + */ const char *availmem( @@ -4462,7 +4484,7 @@ const char *availmem( rm_errno = RM_ERR_SYSTEM; return(NULL); - } /* END availmem() */ + } if (LOGLEVEL >= 6) { @@ -4482,7 +4504,6 @@ const char *availmem( - const char *ncpus( struct rm_attribute *attrib) @@ -4588,9 +4609,13 @@ int find_file( +/* + * physmem() - Determines the amount of RAM on this machine + * + * @return - the amount of RAM in kb as a string in the format: amount, or NULL on error + */ - -static const char *physmem( +const char *physmem( struct rm_attribute *attrib) @@ -4690,6 +4715,10 @@ static const char *physmem( mem_total += mem; } + if ((max_memory > 0) && + (max_memory < mem_total)) + mem_total = max_memory; + sprintf(ret_string, "%llukb", mem_total); return(ret_string); @@ -4697,9 +4726,6 @@ static const char *physmem( - - - char *size_fs( char *param) diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index ad2ba61a84..00421f9fb4 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -195,6 +195,8 @@ int max_join_job_wait_time = MAX_JOIN_WAIT_TIME; int resend_join_job_wait_time = RESEND_WAIT_TIME; int mom_hierarchy_retry_time = NODE_COMM_RETRY_TIME; std::string presetup_prologue; +unsigned long max_memory = 0; +unsigned long max_swap = 0; @@ -305,6 +307,8 @@ unsigned long setmomhierarchyretrytime(const char *); unsigned long setjobdirectorysticky(const char *); unsigned long setcudavisibledevices(const char *); unsigned long set_presetup_prologue(const char *); +unsigned long set_max_physical_memory(const char *); +unsigned long set_max_swap_memory(const char *); struct specials special[] = { { "force_overwrite", setforceoverwrite}, @@ -390,6 +394,8 @@ struct specials special[] = { { "cuda_visible_devices", setcudavisibledevices}, { "cray_check_rur", setrur }, { "presetup_prologue", set_presetup_prologue}, + { "max_physical_memory", set_max_physical_memory}, + { "max_swap_memory", set_max_swap_memory}, { NULL, NULL } }; @@ -507,6 +513,7 @@ unsigned long setforceoverwrite( return(0); /* error */ } + unsigned long setrur( const char *value) @@ -524,6 +531,46 @@ unsigned long setrur( }/* end setrur() */ + +/* + * set_max_physical_memory() + * + * @param value - the maximum memory value in the format + */ + +unsigned long set_max_physical_memory( + + const char *value) + + { + if (read_mem_value(value, max_memory) != PBSE_NONE) + { + sprintf(log_buffer, "Couldn't parse a memory value from '%s'", value); + log_err(-1, __func__, log_buffer); + return(0); + } + + return(1); + } + + + +unsigned long set_max_swap_memory( + + const char *value) + + { + if (read_mem_value(value, max_swap) != PBSE_NONE) + { + sprintf(log_buffer, "Couldn't parse a memory value from '%s'", value); + log_err(-1, __func__, log_buffer); + return(0); + } + + return(1); + } + + unsigned long setidealload( const char *value) @@ -631,8 +678,6 @@ unsigned long setautoidealload( - - unsigned long setallocparcmd( const char *value) /* I */ @@ -647,8 +692,6 @@ unsigned long setallocparcmd( - - unsigned long setautomaxload( const char *value) @@ -795,8 +838,6 @@ unsigned long setmaxload( - - unsigned long setlogfilemaxsize( const char *value) /* I */ @@ -816,7 +857,6 @@ unsigned long setlogfilemaxsize( - unsigned long setlogfilerolldepth( const char *value) /* I */ @@ -848,7 +888,6 @@ unsigned long setlogdirectory( - unsigned long setlogfilesuffix( const char *value) /* I */ diff --git a/src/test/mom_mach/scaffolding.c b/src/test/mom_mach/scaffolding.c index 1869108e87..01ec798991 100644 --- a/src/test/mom_mach/scaffolding.c +++ b/src/test/mom_mach/scaffolding.c @@ -57,6 +57,8 @@ int numa_index; #endif int job_oom_score_adjust = 0; /* no oom score adjust by default */ int mom_oom_immunize = 0; /* make pbs_mom processes immune? no by default */ +unsigned long max_memory = 0; +unsigned long max_swap = 0; void *get_next_return_value = NULL; diff --git a/src/test/mom_mach/test_uut.c b/src/test/mom_mach/test_uut.c index 618fc15261..ac09220bf7 100644 --- a/src/test/mom_mach/test_uut.c +++ b/src/test/mom_mach/test_uut.c @@ -10,6 +10,8 @@ #include "test_mom_mach.h" #include "pbs_job.h" #include "pbs_error.h" +#include "mom_memory.h" +#include "resmon.h" std::string cg_memory_path; @@ -20,6 +22,10 @@ int overmem_proc(job*, unsigned long long); int overcpu_proc(job*, unsigned long); unsigned long long resi_sum(job*); unsigned long long mem_sum(job*); +proc_mem_t *get_proc_mem(); +const char *physmem (struct rm_attribute *); +const char *totmem (struct rm_attribute *); + double cputfactor; @@ -33,6 +39,50 @@ extern proc_stat_t *proc_array; extern void *get_next_return_value; +extern unsigned long max_memory; +extern unsigned long max_swap; + + +START_TEST(memory_tests) + { + proc_mem_t *pm = get_proc_mem(); + const char *mem_str; + char buf[1024]; + rm_attribute ra; + + if (pm != NULL) + { + // Unset these parameters + max_memory = 0; + max_swap = 0; + + mem_str = physmem(&ra); + fail_unless(mem_str != NULL); + sprintf(buf, "%llukb", pm->mem_total >> 10); + fail_unless(!strcmp(buf, mem_str)); + + mem_str = totmem(&ra); + fail_unless(mem_str != NULL); + sprintf(buf, "%llukb", (pm->mem_total >> 10) + (pm->swap_total >> 10)); + fail_unless(!strcmp(buf, mem_str)); + + // Make max_memory and max_swap half of the machine's memory and swap + max_memory = pm->mem_total >> 11; + max_swap = pm->swap_total >> 11; + mem_str = physmem(&ra); + fail_unless(mem_str != NULL); + sprintf(buf, "%lukb", max_memory); + fail_unless(!strcmp(buf, mem_str)); + + mem_str = totmem(&ra); + fail_unless(mem_str != NULL); + sprintf(buf, "%lukb", max_memory + max_swap); + fail_unless(!strcmp(buf, mem_str)); + } + } +END_TEST + + START_TEST(test_get_job_sid_from_pid) { int index = 0; @@ -482,6 +532,7 @@ Suite *mom_mach_suite(void) tc_core = tcase_create("test_mem_sum"); tcase_add_test(tc_core, test_mem_sum); + tcase_add_test(tc_core, memory_tests); suite_add_tcase(s, tc_core); return s; From 82cea51145955f46a7b564e19f654159c6430c48 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 28 Feb 2017 11:44:18 -0700 Subject: [PATCH 168/312] TRQ-3714. Fix an issue with the last check in. --- src/resmom/parse_config.c | 1 + src/test/parse_config/scaffolding.c | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index 00421f9fb4..d65d946e41 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -100,6 +100,7 @@ #include "authorized_hosts.hpp" #include "csv.h" #include "json/json.h" +#include "req.hpp" void encode_used(job *pjob, int perm, Json::Value *, tlist_head *phead); void encode_flagged_attrs(job *pjob, int perm, Json::Value *job_info, tlist_head *phead); diff --git a/src/test/parse_config/scaffolding.c b/src/test/parse_config/scaffolding.c index 375dffffe7..34963fa3b0 100644 --- a/src/test/parse_config/scaffolding.c +++ b/src/test/parse_config/scaffolding.c @@ -226,3 +226,13 @@ void authorized_hosts::add_authorized_address(unsigned long addr, unsigned short authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; +int read_mem_value( + + const char *value, + unsigned long &parsed) + + { + parsed = 10; + return(0); + } + From 9dafb3ee4b41c9b614fa23a1430f56bb88833fef Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 28 Feb 2017 16:14:21 -0700 Subject: [PATCH 169/312] Fix a unit test issue. --- src/test/mail_throttler/test_uut.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/mail_throttler/test_uut.c b/src/test/mail_throttler/test_uut.c index 02ad899625..fc597827ac 100644 --- a/src/test/mail_throttler/test_uut.c +++ b/src/test/mail_throttler/test_uut.c @@ -27,6 +27,7 @@ START_TEST(test_job_constructor) pjob.ji_wattr[JOB_ATR_join].at_flags = ATR_VFLAG_SET; pjob.ji_wattr[JOB_ATR_join].at_val.at_str = strdup(join); pjob.ji_wattr[JOB_ATR_outpath].at_val.at_str = strdup(outpath); + pjob.ji_wattr[JOB_ATR_init_work_dir].at_flags = ATR_VFLAG_SET; pjob.ji_wattr[JOB_ATR_init_work_dir].at_val.at_str = strdup(outpath); pjob.ji_wattr[JOB_ATR_resource].at_val.at_ptr = &resources; pjob.ji_wattr[JOB_ATR_resc_used].at_val.at_ptr = &resources; @@ -43,7 +44,7 @@ START_TEST(test_job_constructor) fail_unless(mi.jobname == jname); fail_unless(mi.queue_name == queue); fail_unless(mi.owner == owner); - fail_unless(mi.working_directory == outpath); + fail_unless(mi.working_directory == outpath, "Got '%s', expected '%s'", mi.working_directory.c_str(), outpath); } END_TEST From 513c77c8d2b71125030e666fc28222b203a4b28c Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 28 Feb 2017 16:45:13 -0700 Subject: [PATCH 170/312] Remove some unnecessary work when sending mom updates to pbs_server. --- src/include/mom_memory.h | 1 + src/resmom/linux/mom_mach.c | 96 +++++------------------------- src/resmom/mom_server.c | 101 +++----------------------------- src/test/mom_mach/scaffolding.c | 1 + 4 files changed, 25 insertions(+), 174 deletions(-) diff --git a/src/include/mom_memory.h b/src/include/mom_memory.h index 2e7891283e..7c872a0017 100644 --- a/src/include/mom_memory.h +++ b/src/include/mom_memory.h @@ -14,6 +14,7 @@ unsigned long long swap_total; unsigned long long swap_used; unsigned long long swap_free; + time_t timestamp; }proc_mem_t; proc_mem_t *get_proc_mem_from_path(const char *path); diff --git a/src/resmom/linux/mom_mach.c b/src/resmom/linux/mom_mach.c index 542f1d27e1..cb796da78f 100644 --- a/src/resmom/linux/mom_mach.c +++ b/src/resmom/linux/mom_mach.c @@ -769,6 +769,11 @@ proc_mem_t *get_proc_mem(void) #else proc_mem_t *mem; #endif + + time_now = time(NULL); + + if (time_now - ret_mm.timestamp < ServerStatUpdateInterval) + return(&ret_mm); #ifdef NUMA_SUPPORT ret_mm.mem_total = 0; @@ -810,6 +815,8 @@ proc_mem_t *get_proc_mem(void) free(mem); #endif + ret_mm.timestamp = time_now; + return(&ret_mm); } /* END get_proc_mem() */ @@ -4620,18 +4627,8 @@ const char *physmem( struct rm_attribute *attrib) { - char tmpBuf[PMEMBUF_SIZE]; - - char *tmp_ptr; - char *BPtr; - int BSpace; - - unsigned long long mem; - unsigned long long mem_total; - FILE *fp; -#ifdef NUMA_SUPPORT - int i; -#endif + proc_mem_t *mm; + unsigned long long mem_total; if (attrib != NULL) { @@ -4642,79 +4639,16 @@ const char *physmem( return(NULL); } - mem_total = 0; - -#ifdef NUMA_SUPPORT - - for (i = 0; i < node_boards[numa_index].num_nodes; i++) -#endif /* NUMA_SUPPORT */ + if ((mm = get_proc_mem()) == NULL) { -#ifdef NUMA_SUPPORT - if (!(fp = fopen(node_boards[numa_index].path_meminfo[i],"r"))) -#else - if (!(fp = fopen(path_meminfo, "r"))) -#endif - { - rm_errno = RM_ERR_SYSTEM; - - return(NULL); - } - - BPtr = tmpBuf; - - BSpace = sizeof(tmpBuf); - - BPtr[0] = '\0'; - - while (!feof(fp)) - { - if (fgets(BPtr, BSpace, fp) == NULL) - { - break; - } - - BSpace -= strlen(BPtr); - - BPtr += strlen(BPtr); - } - - fclose(fp); - - /* FORMAT: '...\nMemTotal: XXX kB\n' */ - - if ((tmp_ptr = strstr(tmpBuf, "MemTotal:")) != NULL) - { - BPtr = tmp_ptr + strlen("MemTotal:"); - - if (sscanf(BPtr, "%llu", - &mem) != 1) - { - rm_errno = RM_ERR_SYSTEM; - - return(NULL); - } - - /* value specified in kb */ - } - else - { - /* attempt to load first numeric value */ - - if (sscanf(BPtr, "%*s %llu", - &mem) != 1) - { - rm_errno = RM_ERR_SYSTEM; - - return(NULL); - } - - /* value specified in bytes */ + log_err(errno, __func__, "get_proc_mem"); - mem >>= 10; - } - mem_total += mem; + rm_errno = RM_ERR_SYSTEM; + return(NULL); } + mem_total = mm->mem_total >> 10; // Convert from bytes to kb + if ((max_memory > 0) && (max_memory < mem_total)) mem_total = max_memory; diff --git a/src/resmom/mom_server.c b/src/resmom/mom_server.c index f4abc5507b..493bd7ce6b 100644 --- a/src/resmom/mom_server.c +++ b/src/resmom/mom_server.c @@ -1764,11 +1764,8 @@ int send_status_through_hierarchy() void mom_server_all_update_stat(void) { - pid_t pid; - int fd_pipe[2]; - int rc; + int rc = -1; char buf[LOCAL_LOG_BUF_SIZE]; - size_t len; time_now = time(NULL); @@ -1799,16 +1796,10 @@ void mom_server_all_update_stat(void) { generate_alps_status(mom_status, apbasil_path, apbasil_protocol); - if (send_update_to_a_server() == PBSE_NONE) - { - ForceServerUpdate = false; - LastServerUpdateTime = time_now; - } + send_update_to_a_server(); } else { - /* The NVIDIA NVML library has a problem when we use it after the first fork. Let's get the gpu status first - and then fork */ #ifdef NVIDIA_GPUS global_gpu_status.clear(); add_gpu_status(global_gpu_status); @@ -1818,103 +1809,27 @@ void mom_server_all_update_stat(void) check_for_mics(global_mic_count); #endif - /* It is possible that pbs_server may get busy and start queing incoming requests and not be able - to process them right away. If pbs_mom is waiting for a reply to a statuys update that has - been queued and at the same time the server makes a request to the mom we can get stuck - in a pseudo live-lock state. That is the server is waiting for a response from the mom and - the mom is waiting for a response from the server. neither of which will come until a request times out. - If we fork the status updates this alleviates the problem by making one less request from the - mom single threaded */ - rc = pipe(fd_pipe); - if (rc != 0) - { - sprintf(buf, "pipe creation failed: %d", errno); - log_err(-1, __func__, buf); - } - - pid = fork(); - - if (pid < 0) - { - log_record(PBSEVENT_SYSTEM, 0, __func__, "Failed to fork stat update process"); - return; - } - - if (pid > 0) - { - // PARENT - close(fd_pipe[1]); - ForceServerUpdate = false; - LastServerUpdateTime = time_now; - UpdateFailCount = 0; - updates_waiting_to_send = 0; - - received_node *rn; - received_statuses.lock(); - container::item_container::item_iterator *iter = received_statuses.get_iterator(); - - // clear cached statuses from hierarchy - while ((rn = iter->get_next_item()) != NULL) - rn->statuses.clear(); - - delete iter; - received_statuses.unlock(); - - len = read(fd_pipe[0], buf, LOCAL_LOG_BUF_SIZE); - - close(fd_pipe[0]); - - if (len <= 0) - { - log_err(-1, __func__, "read of pipe failed for status update"); - return; - } - - if (buf[0] != '0') - num_stat_update_failures++; - else - { - force_layout_update = false; - - num_stat_update_failures = 0; - for (int sindex = 0; sindex < PBS_MAXSERVER; sindex++) - { - if (mom_servers[sindex].pbs_servername[0] == '\0') - continue; - mom_servers[sindex].MOMLastSendToServerTime = time_now; - } - } - - return; - } - - // CHILD - close(fd_pipe[0]); - #ifdef NUMA_SUPPORT for (numa_index = 0; numa_index < num_node_boards; numa_index++) #endif /* NUMA_SUPPORT */ { update_mom_status(); - if (send_status_through_hierarchy() != PBSE_NONE) + if ((rc = send_status_through_hierarchy()) != PBSE_NONE) rc = send_update_to_a_server(); } - sprintf(buf, "%d", rc); - len = strlen(buf); - write(fd_pipe[1], buf, len); - - exit_called = true; - - exit(0); + if (rc == PBSE_NONE) + { + updates_waiting_to_send = 0; + force_layout_update = false; + } } } /* END mom_server_all_update_stat() */ - /** * power */ diff --git a/src/test/mom_mach/scaffolding.c b/src/test/mom_mach/scaffolding.c index 01ec798991..4b5c1cc03b 100644 --- a/src/test/mom_mach/scaffolding.c +++ b/src/test/mom_mach/scaffolding.c @@ -26,6 +26,7 @@ extern std::string cg_memory_path; char mom_alias[PBS_MAXHOSTNAME + 1]; +int ServerStatUpdateInterval = DEFAULT_SERVER_STAT_UPDATES; Machine this_node; std::string cg_cpuacct_path; std::list alljobs_list; From ff5129d89e62a381d4dd172b0dc023ae16aff9da Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 28 Feb 2017 17:27:17 -0700 Subject: [PATCH 171/312] Fix a scaffolding issue. --- src/test/mom_mach/scaffolding.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/mom_mach/scaffolding.c b/src/test/mom_mach/scaffolding.c index 4b5c1cc03b..e8ff8aa547 100644 --- a/src/test/mom_mach/scaffolding.c +++ b/src/test/mom_mach/scaffolding.c @@ -26,7 +26,7 @@ extern std::string cg_memory_path; char mom_alias[PBS_MAXHOSTNAME + 1]; -int ServerStatUpdateInterval = DEFAULT_SERVER_STAT_UPDATES; +int ServerStatUpdateInterval = 45; Machine this_node; std::string cg_cpuacct_path; std::list alljobs_list; From f907f9784cb5afcfdf58c5603cf7e3e58d8411a1 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 1 Mar 2017 09:28:13 -0700 Subject: [PATCH 172/312] Fix another unit test issue. --- src/test/mom_server/test_uut.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/mom_server/test_uut.c b/src/test/mom_server/test_uut.c index 7e54052eb3..0df4eb051f 100644 --- a/src/test/mom_server/test_uut.c +++ b/src/test/mom_server/test_uut.c @@ -208,11 +208,13 @@ START_TEST(test_mom_server_all_update_stat_clear_force) is_reporter_mom = false; + mom_servers[0].MOMLastSendToServerTime = 0; LastServerUpdateTime = time(NULL) - 100; ForceServerUpdate = true; mom_server_all_update_stat(); fail_unless(ForceServerUpdate == false); + mom_servers[0].MOMLastSendToServerTime = 0; LastServerUpdateTime = time(NULL) - 100; ForceServerUpdate = true; mom_server_all_update_stat(); From cfbdae19fd90128a751433b04343f56774193a1a Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 1 Mar 2017 11:53:14 -0700 Subject: [PATCH 173/312] TRQ-3925. Don't allow the specification of both --disable-cpuset and --enable-geometry-requests --- configure.ac | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/configure.ac b/configure.ac index 6e2a863f59..d83745552e 100644 --- a/configure.ac +++ b/configure.ac @@ -991,6 +991,7 @@ else AC_MSG_RESULT([no]) RPM_AC_OPTS="$RPM_AC_OPTS --without cpuset" fi + AM_CONDITIONAL([BUILD_L26_CPUSETS], test "$build_l26_cpuset" = yes) @@ -1016,6 +1017,16 @@ dnl AC_DEFINE(GEOMETRY_REQUESTS, 1, [Define to allow job specific geometry re fi fi + +dnl Don't allow the specification of --disable-cpuset and --enable-geometry-requests +case "${ac_configure_args}" in + *disable-cpuset*) + case "${ac_configure_args}" in + *enable-geometry-requests*) + AC_MSG_ERROR([--enable-geometry-requests requires the use of cpusets. You may not specify both --disable-cpuset and --enable-geometry-requests]) + esac +esac + dnl compile for NUMA systems - allows the site to configure: dnl 1. cpusets and multi-moms dnl 2. memory fences to only access close memory From c9e3af2a4ab1e6f04db073f46f770b46090bc407 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 1 Mar 2017 14:29:07 -0700 Subject: [PATCH 174/312] TRQ-3827. Make the cgroups limit RSS memory. --- src/resmom/linux/mom_mach.c | 17 ++++++-- src/resmom/trq_cgroups.c | 80 ++++++++++++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 14 deletions(-) diff --git a/src/resmom/linux/mom_mach.c b/src/resmom/linux/mom_mach.c index cb796da78f..acaf81d467 100644 --- a/src/resmom/linux/mom_mach.c +++ b/src/resmom/linux/mom_mach.c @@ -1607,7 +1607,7 @@ unsigned long long resi_sum( } } - full_cgroup_path = cg_memory_path + pjob->ji_qs.ji_jobid + "/memory.max_usage_in_bytes"; + full_cgroup_path = cg_memory_path + pjob->ji_qs.ji_jobid + "/memory.stat"; fd = open(full_cgroup_path.c_str(), O_RDONLY); if (fd <= 0) @@ -1631,10 +1631,19 @@ unsigned long long resi_sum( } else if (rc != 0) { - int hardwareStyle; - unsigned long long mem_read; + int hardwareStyle; + unsigned long long mem_read; + char *buf2; + buf2 = strstr(buf, "\nrss "); + if (buf2 == NULL) + { + sprintf(buf, "read failed finding rss %s", full_cgroup_path.c_str()); + log_err(errno, __func__, buf); + close(fd); + return(0); + } - mem_read = strtoull(buf, NULL, 10); + mem_read = strtoull(buf2 + 5, NULL, 10); hardwareStyle = this_node.getHardwareStyle(); diff --git a/src/resmom/trq_cgroups.c b/src/resmom/trq_cgroups.c index 218b60c332..6c2dffeb1b 100644 --- a/src/resmom/trq_cgroups.c +++ b/src/resmom/trq_cgroups.c @@ -777,6 +777,70 @@ int trq_cg_add_process_to_task_cgroup( +/* + * trq_cg_read_numeric_rss() + * + * Reads and returns the rss value from a memory cgroup stat file. + * @param path - the path to the memory cgroup stat file. + * @param error (O) - set to true if there was an error reading the file. + * @return the RSS memory size recorded in this file, or 0 if none could be read. + */ + +unsigned long long trq_cg_read_numeric_rss( + + string &path, + bool &error) + + { + int fd; + unsigned long long val = 0; + char buf[LOCAL_LOG_BUF_SIZE]; + char *buf2; + + error = false; + + fd = open(path.c_str(), O_RDONLY); + if (fd <= 0) + { + // If we don't have a file to open, just return 0 + if (errno == ENOENT) + return(0); + + sprintf(log_buffer, "failed to open %s: %s", path.c_str(), strerror(errno)); + log_err(errno, __func__, log_buffer); + error = true; + } + else + { + int rc = read(fd, buf, LOCAL_LOG_BUF_SIZE); + + close(fd); + + if (rc == -1) + { + sprintf(buf, "read failed getting value from %s - %s", path.c_str(), strerror(errno)); + log_err(errno, __func__, buf); + error = true; + } + else if (rc != 0) + { + buf2=strstr(buf, "\nrss "); + if (buf2 == NULL) + { + sprintf(buf, "read failed finding rss %s", path.c_str()); + log_err(errno, __func__, buf); + error = true; + } + else + val = strtoull(buf2+5, NULL, 10); + } + } + + return(val); + } // END trq_cg_read_numeric_rss() + + + unsigned long long trq_cg_read_numeric_value( string &path, @@ -849,21 +913,17 @@ int trq_cg_get_task_memory_stats( char req_and_task[256]; int rc = PBSE_NONE; - /* get memory first */ - sprintf(req_and_task, "/%s/R%u.t%u/memory.max_usage_in_bytes", job_id, req_index, task_index); + // get memory first + // We read memory.stat instead of memory.max_usage_in_bytes because the latter counts way more than just + // rss memory. trq_cg_read_numeric_rss() gets us just the resident memory size, which is what we really + // want to limit. + sprintf(req_and_task, "/%s/R%u.t%u/memory.stat", job_id, req_index, task_index); string cgroup_path = cg_memory_path + req_and_task; bool error; - mem_used = trq_cg_read_numeric_value(cgroup_path, error); + mem_used = trq_cg_read_numeric_rss(cgroup_path, error); -/* if (mem_used == 0) - { - sprintf(log_buffer, "peak memory usage read from '%s' is 0, something appears to be incorrect.", - cgroup_path.c_str()); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - }*/ - if (error) rc = PBSE_SYSTEM; From fd385d855b52aa343ac1a21b989c171301cb3324 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 2 Mar 2017 13:42:28 -0700 Subject: [PATCH 175/312] TRQ-3944. Reject jobs with unknown resources even if it was submitted to a routing queue. --- src/lib/Libattr/attr_fn_resc.c | 12 ++---------- src/server/req_quejob.c | 32 ++++++-------------------------- 2 files changed, 8 insertions(+), 36 deletions(-) diff --git a/src/lib/Libattr/attr_fn_resc.c b/src/lib/Libattr/attr_fn_resc.c index a8c54aaa0b..bea63e4be3 100644 --- a/src/lib/Libattr/attr_fn_resc.c +++ b/src/lib/Libattr/attr_fn_resc.c @@ -173,15 +173,9 @@ int decode_resc( if (prdef == NULL) { - /* - * didn't find resource with matching name, use unknown; - * but return PBSE_UNKRESC in case caller doesn`t wish to - * accept unknown resources - */ - + // Don't accept unknown resources as this strands jobs. rc = PBSE_UNKRESC; - - prdef = svr_resc_def + (svr_resc_size - 1); + return(rc); } prsc = find_resc_entry(patr, prdef); @@ -220,8 +214,6 @@ int decode_resc( - - /* * encode_resc - encode attr of type ATR_TYPE_RESR into attr_extern form * diff --git a/src/server/req_quejob.c b/src/server/req_quejob.c index b2a8acb681..e30acbb59d 100644 --- a/src/server/req_quejob.c +++ b/src/server/req_quejob.c @@ -814,32 +814,12 @@ int decode_attributes_into_job( if (rc != 0) { - if (rc == PBSE_UNKRESC) - { - /* check for RM extension */ - - /* NYI */ - - /* unknown resources not allowed in Exec queue */ - - if (pque->qu_qs.qu_type == QTYPE_Execution) - { - /* FAILURE */ - svr_job_purge(pj); - job_mutex.set_unlock_on_exit(false); - reply_badattr(rc, 1, psatl, preq); - return(rc); - } - } - else - { - /* FAILURE */ - /* any other error is fatal */ - svr_job_purge(pj); - job_mutex.set_unlock_on_exit(false); - reply_badattr(rc, 1, psatl, preq); - return(rc); - } + /* FAILURE */ + /* any other error is fatal */ + svr_job_purge(pj); + job_mutex.set_unlock_on_exit(false); + reply_badattr(rc, 1, psatl, preq); + return(rc); } /* END if (rc != 0) */ psatl = (svrattrl *)GET_NEXT(psatl->al_link); From e9789f662ef1bc12f18951025386c8d0b71fd4da Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 2 Mar 2017 14:18:18 -0700 Subject: [PATCH 176/312] TRQ-3944. Update a unit test. --- src/test/attr_fn_resc/test_uut.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/attr_fn_resc/test_uut.c b/src/test/attr_fn_resc/test_uut.c index 9b44358aae..687c4e3b89 100644 --- a/src/test/attr_fn_resc/test_uut.c +++ b/src/test/attr_fn_resc/test_uut.c @@ -15,7 +15,7 @@ START_TEST(test_one) fail_unless(decode_resc(NULL,NULL,NULL,NULL,0) == PBSE_INTERNAL); fail_unless(decode_resc(&attr,NULL,NULL,NULL,0) == PBSE_UNKRESC); fail_unless(decode_resc(&attr,"attrName",NULL,NULL,0) == PBSE_UNKRESC); - fail_unless(decode_resc(&attr,"attrName","arch",NULL,0) == PBSE_ATTRRO); + fail_unless(decode_resc(&attr,"attrName","arch",NULL,0) == PBSE_UNKRESC); fail_unless(decode_resc(&attr,"notherAttrName","string","Pie",0) == PBSE_ATTRRO); fail_unless(decode_resc(&attr,"notherAttrName","string","Pie",ATR_DFLAG_ACCESS) == 0); fail_unless(decode_resc(&attr,"someLong","long","314179",ATR_DFLAG_ACCESS) == 0); From 4ade77923399c75f553125daab8ebc52a84673e6 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 3 Mar 2017 11:42:50 -0700 Subject: [PATCH 177/312] TRQ-3952. Add momctl -u to get the mom to force an update of its layout on the server. --- src/include/resmon.h | 11 ++++++----- src/momctl/momctl.c | 23 +++++++++++++++++++++-- src/resmom/mom_main.c | 9 +++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/include/resmon.h b/src/include/resmon.h index 6a674092e0..9c8995c716 100644 --- a/src/include/resmon.h +++ b/src/include/resmon.h @@ -106,11 +106,12 @@ struct config #define RM_NPARM 32 /* max number of parameters for child */ -#define RM_CMD_CLOSE 1 -#define RM_CMD_REQUEST 2 -#define RM_CMD_CONFIG 3 -#define RM_CMD_SHUTDOWN 4 -#define RM_CMD_LAYOUT 5 +#define RM_CMD_CLOSE 1 +#define RM_CMD_REQUEST 2 +#define RM_CMD_CONFIG 3 +#define RM_CMD_SHUTDOWN 4 +#define RM_CMD_LAYOUT 5 +#define RM_CMD_UPDATE_LAYOUT 6 #define RM_RSP_OK 100 #define RM_RSP_ERROR 999 diff --git a/src/momctl/momctl.c b/src/momctl/momctl.c index 081dfa8c13..68c75661a5 100644 --- a/src/momctl/momctl.c +++ b/src/momctl/momctl.c @@ -57,7 +57,8 @@ enum MOMCmdEnum momQuery, momReconfig, momShutdown, - momLayout + momLayout, + momUpdateLayout }; enum MOMCmdEnum CmdIndex = momNONE; @@ -126,7 +127,7 @@ int main( char **ArgV) /* I */ { - const char *OptString = "c:Cd:f:h:lp:q:r:sv"; + const char *OptString = "c:Cd:f:h:lp:q:r:svu"; char HostList[65536]; @@ -371,6 +372,12 @@ int main( break; + case 'u': + + CmdIndex = momUpdateLayout; + + break; + case 'v': /* report verbose logging */ @@ -841,6 +848,18 @@ int do_mom( break; + case momUpdateLayout: + + if ((send_command(chan, RM_CMD_UPDATE_LAYOUT) != PBSE_NONE) || + (check_success(chan) != PBSE_NONE)) + { + fprintf(stdout, "Update layout command failed to send to mom\n"); + return(-1); + } + + + break; + case momQuery: default: diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index f6b062f693..6e15c2032b 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -3120,6 +3120,15 @@ int rm_request( break; + case RM_CMD_UPDATE_LAYOUT: + + // Force an over-write of the layout on the next successful send to pbs_server + force_layout_update = true; + diswsi(chan, RM_RSP_OK); + DIS_tcp_wflush(chan); + + break; + default: sprintf(log_buffer, "unknown command %d", From 2b2e290d99baa784782ca9ea896d63424a8da6a9 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 6 Mar 2017 17:28:39 -0700 Subject: [PATCH 178/312] TRQ-3031. Notify the interactive process when a job is canceled while waiting to start. Conflicts: src/server/req_delete.c --- src/cmds/qsub_functions.c | 13 +++++- src/server/req_delete.c | 85 ++++++++++++++++++++++++++++++++++----- 2 files changed, 86 insertions(+), 12 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 815c133642..e94b08ca05 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -2396,6 +2396,10 @@ int get_interactive_job_id( break; } + if ((pc - momjobid == 1) && + (momjobid[0] == 'C')) + return(-2); + // Return -1 if we didn't read anything if (amt == momjobid_size) { @@ -2461,11 +2465,18 @@ void interactive( for (int retry = 0; retry < 3; retry++) { - if (get_interactive_job_id(news, momjobid, sizeof(momjobid)) == PBSE_NONE) + int rc = get_interactive_job_id(news, momjobid, sizeof(momjobid)); + if (rc == PBSE_NONE) { read_jobid = true; break; } + else if (rc == -2) + { + // The job was canceled while waiting to start + fprintf(stderr, "qsub: Job %s was canceled while waiting to start\n", new_jobname); + exit(1); + } } if (read_jobid == false) diff --git a/src/server/req_delete.c b/src/server/req_delete.c index 8ce5c62f58..80892c868e 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -89,6 +89,7 @@ #include +#include #include #include #include @@ -117,8 +118,8 @@ #include "mutex_mgr.hpp" #include "threadpool.h" #include "req_delete.h" +#include "lib_ifl.h" #include "delete_all_tracker.hpp" -#include #define PURGE_SUCCESS 1 #define MOM_DELETE 2 @@ -486,7 +487,63 @@ void setup_apply_job_delete_nanny( /* - * execut_job_delete() + * notify_waiting_terminal() + * + * @param pjob - the interactive job whose terminal we need to notify + */ + +void notify_waiting_terminal( + + job *pjob) + + { + unsigned int pport = pjob->ji_wattr[JOB_ATR_interactive].at_val.at_long; + char *hostname = pjob->ji_wattr[JOB_ATR_submit_host].at_val.at_str; + char log_buf[LOCAL_LOG_BUF_SIZE]; + pbs_net_t hostaddr; + int local_errno; + + if (hostname != NULL) + { + hostaddr = get_hostaddr(&local_errno, hostname); + + if (hostaddr != (pbs_net_t)0) + { + char err_msg[MAXLINE]; + int s = client_to_svr(hostaddr, pport, 0, err_msg); + + if (s >= 0) + { + // Write the letter 'C' to cancel the job + write_ac_socket(s, "C", 1); + close(s); + } + else + { + sprintf(log_buf, "Couldn't open a socket to cancel interactive job '%s' : %s", + pjob->ji_qs.ji_jobid, err_msg); + log_err(errno, __func__, log_buf); + } + } + else + { + sprintf(log_buf, "Couldn't get the address for host '%s' to cancel interactive job '%s'", + hostname, pjob->ji_qs.ji_jobid); + log_err(errno, __func__, log_buf); + } + } + else + { + sprintf(log_buf, "Couldn't find a submission host for interactive job '%s'", pjob->ji_qs.ji_jobid); + log_err(-1, __func__, log_buf); + } + + } // END notify_waiting_terminal() + + + +/* + * execute_job_delete() * * Deletes an individual job, contacting the mom if necessary * @param pjob - the job being deleted @@ -711,18 +768,24 @@ int execute_job_delete( return(-1); } /* END if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) */ - else if ((pjob->ji_qs.ji_state < JOB_STATE_RUNNING) && - (status_cancel_queue != 0)) + else if (pjob->ji_qs.ji_state < JOB_STATE_RUNNING) { - /* - * If a task is not running yet and is still on the queue, we - * need to set an exit status - */ + if (status_cancel_queue != 0) + { + /* + * If a task is not running yet and is still on the queue, we + * need to set an exit status + */ - pjob->ji_qs.ji_un.ji_exect.ji_exitstat = status_cancel_queue; + pjob->ji_qs.ji_un.ji_exect.ji_exitstat = status_cancel_queue; + + pjob->ji_wattr[JOB_ATR_exitstat].at_val.at_long = status_cancel_queue; + pjob->ji_wattr[JOB_ATR_exitstat].at_flags |= ATR_VFLAG_SET; + } - pjob->ji_wattr[JOB_ATR_exitstat].at_val.at_long = status_cancel_queue; - pjob->ji_wattr[JOB_ATR_exitstat].at_flags |= ATR_VFLAG_SET; + // Send cancel to the waiting terminal + if (pjob->ji_wattr[JOB_ATR_interactive].at_val.at_long) + notify_waiting_terminal(pjob); } if (perform_job_delete_array_bookkeeping(pjob, status_cancel_queue) != PBSE_NONE) From 5c1cdc30058f13dab1beb7c9b584de1e169d9363 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 7 Mar 2017 09:33:34 -0700 Subject: [PATCH 179/312] TRQ-3031. Add two scaffolding definitions. Conflicts: src/test/req_delete/scaffolding.c --- src/test/req_delete/scaffolding.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/test/req_delete/scaffolding.c b/src/test/req_delete/scaffolding.c index 298ecb02cf..229bc6a454 100644 --- a/src/test/req_delete/scaffolding.c +++ b/src/test/req_delete/scaffolding.c @@ -547,3 +547,23 @@ batch_request::~batch_request() batch_request::batch_request() {} batch_request::batch_request(int type) : rq_type(type) {} +int client_to_svr( + + pbs_net_t hostaddr, /* I - internet addr of host */ + unsigned int port, /* I - port to which to connect */ + int local_port, /* I - BOOLEAN: not 0 to use local reserved port */ + char *EMsg) /* O (optional,minsize=1024) */ + + { + return(1); + } + +pbs_net_t get_hostaddr( + + int *local_errno, /* O */ + const char *hostname) /* I */ + + { + return(0); + } + From aeec37ff37d774aa83365c9a06a3760029918759 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 7 Mar 2017 15:18:42 -0700 Subject: [PATCH 180/312] TRQ-1264. Add INTERACTIVE_PORT_RANGE to torque.cfg to force a range for interactive ports. --- src/cmds/qsub_functions.c | 54 ++++++++++++++++++++++++++++-- src/test/qsub_functions/test_uut.c | 20 +++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index e94b08ca05..f2667814cf 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -89,9 +89,12 @@ char *host_name_suffix = NULL; int J_opt = FALSE; int P_opt = FALSE; +const int MAX_BIND_RETRIES = 20; const char *checkpoint_strings = "n,c,s,u,none,shutdown,periodic,enabled,interval,depth,dir"; char *alternate_dependency; int alternate_data_type; +int interactive_port_min; +int interactive_port_max; complete_req cr; std::string L_request; @@ -1665,15 +1668,27 @@ void do_dir( } /* END do_dir() */ + +unsigned int get_port_in_range() + + { + int span = interactive_port_max - interactive_port_min; + return(rand() % span + interactive_port_min); + } // END get_port_in_range() + + + /* * The following bunch of functions support the "Interactive Job" * capability of PBS. */ + /* * interactive_port - get a socket to listen to for "interactive" job * When the "interactive" job is run, its standard in, out, and error * will be connected to this socket. */ + char *interactive_port( int *sock) @@ -1684,6 +1699,7 @@ char *interactive_port( struct sockaddr_in myaddr; unsigned short port; + int bind_rc = -1; *sock = socket(AF_INET, SOCK_STREAM, 0); @@ -1696,11 +1712,20 @@ char *interactive_port( myaddr.sin_family = AF_INET; myaddr.sin_addr.s_addr = INADDR_ANY; - if (bind(*sock, (struct sockaddr *)&myaddr, namelen) < 0) + // Since the requested port range could be in use, retry throughout the port range. + for (int bind_retry = 0; bind_retry < MAX_BIND_RETRIES; bind_retry++) + { + if (interactive_port_min != 0) + myaddr.sin_port = htons(get_port_in_range()); + + if ((bind_rc = bind(*sock, (struct sockaddr *)&myaddr, namelen)) >= 0) + break; + } + + if (bind_rc < 0) print_qsub_usage_exit("qsub: unable to bind to socket"); /* get port number assigned */ - if (getsockname(*sock, (struct sockaddr *)&myaddr, &namelen) < 0) print_qsub_usage_exit("qsub: unable to get port number"); @@ -4234,6 +4259,29 @@ void process_config_file( strcpy(host_name_suffix, param_val); } } + + if ((param_val = get_trq_param("INTERACTIVE_PORT_RANGE", torque_cfg_buf)) != NULL) + { + // Must be - + char *ptr = param_val; + bool valid = false; + interactive_port_min = strtol(ptr, &ptr, 10); + if (*ptr == '-') + { + ptr++; + interactive_port_max = strtol(ptr, &ptr, 10); + + if ((interactive_port_max > interactive_port_min) && + (interactive_port_min > 1024)) + valid = true; + } + + if (valid == false) + { + interactive_port_min = 0; + interactive_port_max = 0; + } + } } /* END if (load_config(torque_cfg_buf,sizeof(torque_cfg_buf)) == 0) */ } @@ -4516,6 +4564,8 @@ void main_func( int debug = FALSE; job_info ji; + srand(time(NULL)); + /** * Before we go to the trouble of allocating memory, initializing structures, * and setting up for ordinary workflow, check options to see if we'll be diff --git a/src/test/qsub_functions/test_uut.c b/src/test/qsub_functions/test_uut.c index 39a18bc7f9..46021933f6 100644 --- a/src/test/qsub_functions/test_uut.c +++ b/src/test/qsub_functions/test_uut.c @@ -24,6 +24,7 @@ int process_opt_p(job_info *ji, const char *cmd_arg, int data_type); int wait_for_read_ready(int, int); bool is_memory_request_valid(job_info *ji, std::string &err_msg); int validate_pbs_o_workdir(const job_info *); +unsigned int get_port_in_range(); extern complete_req cr; extern bool mem_fail; @@ -41,6 +42,9 @@ extern int global_poll_rc; extern short global_poll_revents; extern int global_poll_errno; +extern int interactive_port_min; +extern int interactive_port_max; + bool are_we_forking() { @@ -54,6 +58,21 @@ bool are_we_forking() } +START_TEST(test_get_port_in_range) + { + interactive_port_min = 30000; + interactive_port_max = 30100; + + for (int i = 0; i < 100; i++) + { + unsigned int port = get_port_in_range(); + fail_unless(port >= interactive_port_min); + fail_unless(port <= interactive_port_max); + } + } +END_TEST + + START_TEST(test_is_memory_request_valid) { #ifdef PENABLE_LINUX_CGROUPS @@ -489,6 +508,7 @@ Suite *qsub_functions_suite(void) tc_core = tcase_create("test_make_argv"); tcase_add_test(tc_core, test_make_argv); tcase_add_test(tc_core, test_is_resource_request_valid); + tcase_add_test(tc_core, test_get_port_in_range); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_wait_for_read_ready"); From 41b056e95686dc6df1c2e2f6e0fb8c5b3b679e86 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 7 Mar 2017 16:20:37 -0700 Subject: [PATCH 181/312] TRQ-3881. Optimize gaining access to the gpus while they're being used. --- src/resmom/nvidia.c | 144 +++++++++++++++++++++++++++++++++----------- 1 file changed, 109 insertions(+), 35 deletions(-) diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index a6c152b00f..527edeec20 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -654,8 +654,11 @@ static int gpumodes( */ int setgpumode( - int gpuid, - int gpumode) + + int gpuid, + int gpumode, + bool initialized) + { #ifdef NVML_API nvmlReturn_t rc; @@ -663,30 +666,43 @@ int setgpumode( nvmlDevice_t device_hndl; char gpu_id[20]; - if (!init_nvidia_nvml(global_gpu_count)) - return(PBSE_GPU_NOT_INITIALIZED); - - if (!check_nvidia_setup()) + if (initialized == false) { - shut_nvidia_nvml(); - return(PBSE_GPU_NOT_INITIALIZED); + if (!init_nvidia_nvml(global_gpu_count)) + return(PBSE_GPU_NOT_INITIALIZED); + + if (!check_nvidia_setup()) + { + shut_nvidia_nvml(); + return(PBSE_GPU_NOT_INITIALIZED); + } } switch (gpumode) { case gpu_normal: + compute_mode = NVML_COMPUTEMODE_DEFAULT; break; + case gpu_exclusive_thread: + compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_THREAD; break; + case gpu_prohibited: - shut_nvidia_nvml(); + + if (initialized == false) + shut_nvidia_nvml(); + return (PBSE_IVALREQ); break; + case gpu_exclusive_process: + compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_PROCESS; break; + default: if (LOGLEVEL >= 1) { @@ -695,7 +711,9 @@ int setgpumode( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); } - shut_nvidia_nvml(); + if (initialized == false) + shut_nvidia_nvml(); + return (PBSE_IVALREQ); } @@ -718,7 +736,9 @@ int setgpumode( if (rc == NVML_SUCCESS) { - shut_nvidia_nvml(); + if (initialized == false) + shut_nvidia_nvml(); + return (PBSE_NONE); } @@ -728,7 +748,10 @@ int setgpumode( sprintf(log_buffer, "Failed to get device handle for gpu id %d, nvml error %d", gpuid, rc); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - shut_nvidia_nvml(); + + if (initialized == false) + shut_nvidia_nvml(); + return(PBSE_SYSTEM); #else @@ -809,7 +832,7 @@ int setgpumode( return(PBSE_NONE); #endif /* NVML_API */ - } + } // END setgpumode() /* @@ -1005,12 +1028,13 @@ int resetgpuecc( int set_gpu_modes( - std::vector& gpu_indices, - int gpu_flags) + std::vector &gpu_indices, + int gpu_flags) { - int rc = PBSE_NONE; - int gpu_mode = -1; + int rc = PBSE_NONE; + int gpu_mode = -1; + bool initialized = false; gpu_mode = gpu_flags; @@ -1024,6 +1048,20 @@ int set_gpu_modes( gpu_mode -= 1000; } + +#ifdef NVML_API + if (!init_nvidia_nvml(global_gpu_count)) + return(PBSE_GPU_NOT_INITIALIZED); + + if (!check_nvidia_setup()) + { + shut_nvidia_nvml(); + return(PBSE_GPU_NOT_INITIALIZED); + } + + initialized = true; +#endif + for (std::vector::iterator it = gpu_indices.begin(); it != gpu_indices.end(); it++) { int set_mode_result; @@ -1040,7 +1078,7 @@ int set_gpu_modes( } } - set_mode_result = setgpumode(*it, gpu_mode); + set_mode_result = setgpumode(*it, gpu_mode, initialized); if (set_mode_result != PBSE_NONE) { sprintf(log_buffer, "could not set mode on gpu %d", *it); @@ -1048,9 +1086,13 @@ int set_gpu_modes( rc = set_mode_result; } } + +#ifdef NVML_API + shut_nvidia_nvml(); +#endif return(rc); - } + } // END set_gpu_modes() /* @@ -1105,22 +1147,22 @@ int get_gpu_mode( int set_gpu_req_modes( - std::vector& gpu_indices, - int gpu_flags, - job *pjob) + std::vector &gpu_indices, + int gpu_flags, + job *pjob) { pbs_attribute *pattr; - int rc; - unsigned int gpu_indices_size = gpu_indices.size(); - std::vector::iterator it = gpu_indices.begin(); + int rc; + unsigned int gpu_indices_size = gpu_indices.size(); + size_t gpu_index = 0; + bool initialized = false; /* Is the a -l resource request or a -L resource request */ pattr = &pjob->ji_wattr[JOB_ATR_resource]; if (have_incompatible_dash_l_resource(pattr) == true) { rc = set_gpu_modes(gpu_indices, gpu_flags); - } else { @@ -1139,7 +1181,20 @@ int set_gpu_req_modes( req r; std::string gpu_mode; - for (unsigned int i = 0; i < cr->get_num_reqs() && it != gpu_indices.end(); i++) +#ifdef NVML_API + if (!init_nvidia_nvml(global_gpu_count)) + return(PBSE_GPU_NOT_INITIALIZED); + + if (!check_nvidia_setup()) + { + shut_nvidia_nvml(); + return(PBSE_GPU_NOT_INITIALIZED); + } + + initialized = true; +#endif + + for (unsigned int i = 0; i < cr->get_num_reqs() && gpu_index < gpu_indices.size(); i++) { int total_req_gpus; @@ -1148,7 +1203,7 @@ int set_gpu_req_modes( gpu_mode = r.get_gpu_mode(); /* returns a string indicating the gpu mode */ total_req_gpus = r.get_gpus(); - for (unsigned int j = 0; j < total_req_gpus && it != gpu_indices.end(); j++) + for (unsigned int j = 0; j < total_req_gpus && gpu_index < gpu_indices.size(); j++) { /* only use as many gpus as requested for the req */ int mode; @@ -1156,36 +1211,54 @@ int set_gpu_req_modes( if (gpu_flags >= 1000) { /* We need to reset error counts */ - resetgpuecc(*it, 0, 1); + resetgpuecc(gpu_indices[gpu_index], 0, 1); } mode = get_gpu_mode(gpu_mode); if (mode == gpu_mode_not_set) + { +#ifdef NVML_API + shut_nvidia_nvml(); +#endif return(PBSE_NONE); + } if (mode == gpu_unknown) { sprintf(log_buffer, "gpu mode unknown: %s", pjob->ji_qs.ji_jobid); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + +#ifdef NVML_API + shut_nvidia_nvml(); +#endif return(PBSE_IVALREQ); } - rc = setgpumode(*it, mode); + rc = setgpumode(gpu_indices[gpu_index], mode, initialized); if (rc != PBSE_NONE) { - sprintf(log_buffer, "could not set mode on gpu %d", *it); + sprintf(log_buffer, "could not set mode on gpu %d", gpu_indices[gpu_index]); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + +#ifdef NVML_API + shut_nvidia_nvml(); +#endif + return(PBSE_IVALREQ); } - it++; + + gpu_index++; } } - - rc = PBSE_NONE; + rc = PBSE_NONE; } } +#ifdef NVML_API + shut_nvidia_nvml(); +#endif + return(rc); } @@ -2106,6 +2179,7 @@ void generate_server_gpustatus_smi( } + void req_gpuctrl_mom( struct batch_request *preq) /* I */ @@ -2164,7 +2238,7 @@ void req_gpuctrl_mom( if (gpumode != -1) { - rc = setgpumode(gpuid, gpumode); + rc = setgpumode(gpuid, gpumode, false); } else if ((reset_perm != -1) || (reset_vol != -1)) { From bd61a980a8924408079dd6c6f8f791927c6636aa Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 8 Mar 2017 11:16:36 -0700 Subject: [PATCH 182/312] TRQ-3885. default_gpu_mode wasn't working after a restart. --- src/include/policy_values.h | 3 ++ src/server/pbsd_init.c | 8 +++-- src/server/req_manager.c | 64 ++++++++++++++++++++++--------------- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/src/include/policy_values.h b/src/include/policy_values.h index ec6a179227..b4ed28810a 100644 --- a/src/include/policy_values.h +++ b/src/include/policy_values.h @@ -2,4 +2,7 @@ extern bool cray_enabled; extern bool ghost_array_recovery; +extern int default_gpu_mode; + +int set_default_gpu_mode_int(const char *gpu_mode_str); diff --git a/src/server/pbsd_init.c b/src/server/pbsd_init.c index db25179df8..d5aec1a227 100644 --- a/src/server/pbsd_init.c +++ b/src/server/pbsd_init.c @@ -137,6 +137,7 @@ #include "id_map.hpp" #include "exiting_jobs.h" #include "mom_hierarchy_handler.h" +#include "policy_values.h" /*#ifndef SIGKILL*/ @@ -2220,8 +2221,9 @@ void setup_threadpool() void set_server_policies() { - bool cray = false; - bool recover_subjobs = false; + bool cray = false; + bool recover_subjobs = false; + char *default_gpu_str = NULL; if (get_svr_attr_b(SRV_ATR_CrayEnabled, &cray) == PBSE_NONE) cray_enabled = cray; @@ -2229,6 +2231,8 @@ void set_server_policies() if (get_svr_attr_b(SRV_ATR_GhostArrayRecovery, &recover_subjobs) == PBSE_NONE) ghost_array_recovery = recover_subjobs; + if (get_svr_attr_str(SRV_ATR_DefaultGpuMode, &default_gpu_str) == PBSE_NONE) + set_default_gpu_mode_int(default_gpu_str); } // END set_server_policies() diff --git a/src/server/req_manager.c b/src/server/req_manager.c index 45d8af2129..48df4a078d 100644 --- a/src/server/req_manager.c +++ b/src/server/req_manager.c @@ -2702,6 +2702,42 @@ int extra_resc_chk( +int set_default_gpu_mode_int( + + const char *gpu_mode_str) + + { + int rc = PBSE_NONE; + + // Make sure it's one of the acceptable gpu modes + if (!strcmp(gpu_mode_str, "exclusive_thread")) + { + default_gpu_mode = gpu_exclusive_thread; + } + else if (!strcmp(gpu_mode_str, "exclusive")) + { + default_gpu_mode = gpu_exclusive; + } + else if (!strcmp(gpu_mode_str, "exclusive_process")) + { + default_gpu_mode = gpu_exclusive_process; + } + else if (!strcmp(gpu_mode_str, "default")) + { + default_gpu_mode = gpu_normal; + } + else if (!strcmp(gpu_mode_str, "shared")) + { + default_gpu_mode = gpu_normal; + } + else + rc = PBSE_ATTRTYPE; + + return(rc); + } // END set_default_gpu_mode_int() + + + /* * check_default_gpu_mode_str() * @@ -2725,33 +2761,9 @@ int check_default_gpu_mode_str( // this shouldn't happen if (gpu_mode == NULL) - { - return(PBSE_ATTRTYPE); - } - - // Make sure it's one of the acceptable gpu modes - if (!strcmp(gpu_mode, "exclusive_thread")) - { - default_gpu_mode = gpu_exclusive_thread; - } - else if (!strcmp(gpu_mode, "exclusive")) - { - default_gpu_mode = gpu_exclusive; - } - else if (!strcmp(gpu_mode, "exclusive_process")) - { - default_gpu_mode = gpu_exclusive_process; - } - else if (!strcmp(gpu_mode, "default")) - { - default_gpu_mode = gpu_normal; - } - else if (!strcmp(gpu_mode, "shared")) - { - default_gpu_mode = gpu_normal; - } - else rc = PBSE_ATTRTYPE; + else + rc = set_default_gpu_mode_int(gpu_mode); break; } From 7a71de2e4dad35d73e4c3f55af32564cac17986f Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 8 Mar 2017 11:21:55 -0700 Subject: [PATCH 183/312] TRQ-3885. Add two scaffolding definitions. --- src/test/pbsd_init/scaffolding.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/test/pbsd_init/scaffolding.c b/src/test/pbsd_init/scaffolding.c index 096c70617a..24ee10ff4e 100644 --- a/src/test/pbsd_init/scaffolding.c +++ b/src/test/pbsd_init/scaffolding.c @@ -990,3 +990,17 @@ void job_array::update_array_values( { } + +int set_default_gpu_mode_int(const char *default_gpu_str) + { + return(0); + } + +int get_svr_attr_str( + + int attr_index, + char **str) + + { + return(0); + } From d003e4fd27178d55bbae1c3d292510ca8851209b Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 8 Mar 2017 16:48:13 -0700 Subject: [PATCH 184/312] TRQ-3885. "exclusive" is not a valid gpu mode. --- src/server/req_manager.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/server/req_manager.c b/src/server/req_manager.c index 48df4a078d..18ef19f7f7 100644 --- a/src/server/req_manager.c +++ b/src/server/req_manager.c @@ -2714,10 +2714,6 @@ int set_default_gpu_mode_int( { default_gpu_mode = gpu_exclusive_thread; } - else if (!strcmp(gpu_mode_str, "exclusive")) - { - default_gpu_mode = gpu_exclusive; - } else if (!strcmp(gpu_mode_str, "exclusive_process")) { default_gpu_mode = gpu_exclusive_process; From ab0a816180a2d301b90fdc99d7cc56a073e25096 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 8 Mar 2017 17:04:07 -0700 Subject: [PATCH 185/312] TRQ-3885. Update the unit test. --- src/test/req_manager/test_uut.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/req_manager/test_uut.c b/src/test/req_manager/test_uut.c index 7a44842be4..966ee1579b 100644 --- a/src/test/req_manager/test_uut.c +++ b/src/test/req_manager/test_uut.c @@ -19,15 +19,15 @@ START_TEST(test_check_default_gpu_mode_str) pattr.at_val.at_str = strdup("exclusive_process"); fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); fail_unless(default_gpu_mode == gpu_exclusive_process); + pattr.at_val.at_str = strdup("exclusive_thread"); fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); fail_unless(default_gpu_mode == gpu_exclusive_thread); - pattr.at_val.at_str = strdup("exclusive"); - fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); - fail_unless(default_gpu_mode == gpu_exclusive); + pattr.at_val.at_str = strdup("default"); fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); fail_unless(default_gpu_mode == gpu_normal); + pattr.at_val.at_str = strdup("shared"); fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); fail_unless(default_gpu_mode == gpu_normal); From 9ce2dc64bea4e5bc606f16ccdc7a5d26630549d4 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 9 Mar 2017 11:12:16 -0700 Subject: [PATCH 186/312] TRQ-3937. Add a unit test to verify this. --- src/include/pbs_job.h | 1 + src/server/job.cpp | 6 ++++++ src/test/job/test_uut.c | 7 +++++++ 3 files changed, 14 insertions(+) diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 32d255ee38..080c32e69f 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -843,6 +843,7 @@ class job void encode_plugin_resource_usage(tlist_head *phead) const; void add_plugin_resource_usage(std::string &acct_data) const; + size_t number_of_plugin_resources() const; }; #endif diff --git a/src/server/job.cpp b/src/server/job.cpp index da3372c989..3024161083 100644 --- a/src/server/job.cpp +++ b/src/server/job.cpp @@ -192,3 +192,9 @@ void job::set_plugin_resource_usage_from_json( if (reader.parse(json_str, resources) == true) this->set_plugin_resource_usage_from_json(resources); } + + +size_t job::number_of_plugin_resources() const + { + return(this->ji_plugin_usage_info.size()); + } diff --git a/src/test/job/test_uut.c b/src/test/job/test_uut.c index 95921f525c..729c4ad6e8 100644 --- a/src/test/job/test_uut.c +++ b/src/test/job/test_uut.c @@ -62,6 +62,13 @@ START_TEST(test_plugin_things) fail_unless(acct_data.find("resources_used.breaths=10000") != std::string::npos); fail_unless(acct_data.find("resources_used.aons_drawn=20") != std::string::npos); + job pjob2; + Json::Value p2; + p2["bob"]; + p2["name"] = "tom"; + pjob2.set_plugin_resource_usage_from_json(p2); + fail_unless(pjob2.number_of_plugin_resources() == 1); + } END_TEST From 538263ca945a5ccf22e1a7d44c9a5fda9fc1fff5 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 9 Mar 2017 15:06:53 -0700 Subject: [PATCH 187/312] TRQ-3524. Make sisters clean up a job even when mother superior no longer remembers the job. Conflicts: src/test/mom_comm/scaffolding.c --- src/resmom/mom_comm.c | 46 ++++++++++++++++++++++++++++++++- src/test/mom_comm/scaffolding.c | 2 ++ src/test/mom_comm/test_uut.c | 14 ++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index 14a4bf11e5..7883fefca0 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -4980,7 +4980,31 @@ void create_contact_list( if (node_addr != ipaddr_connect) sister_list.insert(i); } - } + } // END create_contact_list() + + + +/* + * process_kill_or_abort_error_from_ms() + * + * I sent a kill or abort to mother superior and got an error back; mother superior + * must've already cleaned up the job, so I should too. + * + * @param pjob - the job we're about to delete + * @return PBSE_NONE + */ + +int process_kill_or_abort_error_from_ms( + + job *pjob) + + { + int rc = PBSE_NONE; + + mom_deljob(pjob); + + return(rc); + } // END process_kill_or_abort_error_from_ms() @@ -5158,6 +5182,22 @@ int im_poll_error( +/* + * is_mother_superior() + * + * @return true if np describes mother superior for its job, false otherwise + */ + +bool is_mother_superior( + + hnodent *np) + + { + return(np->hn_node == 0); + } // END is_mother_superior() + + + /* * process_error_reply @@ -5247,6 +5287,10 @@ int process_error_reply( case IM_KILL_JOB: rc = process_end_job_error_reply(pjob, np, pSockAddr, errcode); + + if ((rc == IM_FAILURE) && + (is_mother_superior(np) == true)) + rc = process_kill_or_abort_error_from_ms(pjob); break; diff --git a/src/test/mom_comm/scaffolding.c b/src/test/mom_comm/scaffolding.c index 705b4a610f..023150a209 100644 --- a/src/test/mom_comm/scaffolding.c +++ b/src/test/mom_comm/scaffolding.c @@ -677,3 +677,5 @@ void authorized_hosts::list_authorized_hosts(std::string &output) {} authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; +void mom_deljob(job *pjob) {} + diff --git a/src/test/mom_comm/test_uut.c b/src/test/mom_comm/test_uut.c index ce0f65b892..08dfdd6eda 100644 --- a/src/test/mom_comm/test_uut.c +++ b/src/test/mom_comm/test_uut.c @@ -42,6 +42,7 @@ received_node *get_received_node_entry(char *str); bool is_nodeid_on_this_host(job *pjob, tm_node_id nodeid); task *find_task_by_pid(job *pjob, int pid); int readit(int, int); +bool is_mother_superior(hnodent *np); #ifdef PENABLE_LINUX_CGROUPS int get_req_and_task_index_from_local_rank(job *pjob, int local_rank, unsigned int &req_index, unsigned int &task_index); @@ -74,6 +75,18 @@ END_TEST #endif +START_TEST(test_is_mother_superior) + { + hnodent hn; + + hn.hn_node = 1; + fail_unless(is_mother_superior(&hn) == false); + hn.hn_node = 0; + fail_unless(is_mother_superior(&hn) == true); + } +END_TEST + + START_TEST(test_get_reply_stream) { job pjob; @@ -646,6 +659,7 @@ Suite *mom_comm_suite(void) tc_core = tcase_create("test_readit"); tcase_add_test(tc_core, test_readit); + tcase_add_test(tc_core, test_is_mother_superior); suite_add_tcase(s, tc_core); return(s); From 538bc32af8a5cfdad7a1d356b3e72d3c873e93c8 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 9 Mar 2017 16:47:05 -0700 Subject: [PATCH 188/312] TRQ-3950. Fix a potential hang if a job somehow gets a dependency on itself. Conflicts: src/include/pbs_job.h src/server/job_func.c src/server/req_register.c src/test/req_register/scaffolding.c --- src/include/pbs_job.h | 1 + src/server/job_func.c | 12 ++++++++++++ src/server/req_register.c | 11 +++++++++++ src/test/req_register/scaffolding.c | 20 ++++++++++++++++++++ src/test/req_register/test_uut.c | 16 ++++++++++++---- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 080c32e69f..1b3a183f78 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -1310,6 +1310,7 @@ job *svr_find_job(const char *jobid, int get_subjob); job *svr_find_job_by_id(int internal_job_id); job *find_job_by_array(all_jobs *aj, const char *job_id, int get_subjob, bool locked); bool job_id_exists(const std::string &job_id_string); +void *svr_job_purge_task(void *vp); bool internal_job_id_exists(int internal_id); #else extern job *mom_find_job(const char *); diff --git a/src/server/job_func.c b/src/server/job_func.c index c8950a3650..8240e6131e 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -2092,6 +2092,18 @@ int svr_job_purge( +void *svr_job_purge_task( + + void *vp) + + { + job *pjob = (job *)vp; + svr_job_purge(pjob); + return(NULL); + } // END svr_job_purge_task() + + + /* * Always access the array in this way in order to avoid deadlock * diff --git a/src/server/req_register.c b/src/server/req_register.c index 4257863e21..974fded556 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -107,6 +107,7 @@ #include "mutex_mgr.hpp" #include "utils.h" #include "job_func.h" +#include "threadpool.h" #define SYNC_SCHED_HINT_NULL 0 @@ -1889,6 +1890,7 @@ void set_depend_hold( int loop = 1; int newstate; int newsubst; + char log_buf[LOCAL_LOG_BUF_SIZE]; struct depend *pdp = NULL; depend_job *djob = NULL; @@ -1941,6 +1943,15 @@ void set_depend_hold( lock the job. It is already locked */ if (!jobids_match) djp = svr_find_job(djob->dc_child.c_str(), TRUE); + else + { + // This should never happen. A Job cannot depend on itself. + snprintf(log_buf, sizeof(log_buf), + "Job %s somehow has a dependency on itself. Purging.", pjob->ji_qs.ji_jobid); + log_err(-1, __func__, log_buf); + enqueue_threadpool_request(svr_job_purge_task, pjob, task_pool); + throw (int)PBSE_BADDEPEND; + } if (!djp || ((pdp->dp_type == JOB_DEPEND_TYPE_AFTERSTART) && diff --git a/src/test/req_register/scaffolding.c b/src/test/req_register/scaffolding.c index dd4310bcd4..27333d1efe 100644 --- a/src/test/req_register/scaffolding.c +++ b/src/test/req_register/scaffolding.c @@ -11,6 +11,7 @@ #include "array.h" /* job_array */ #include "work_task.h" /* work_task */ #include "queue.h" +#include "threadpool.h" const int DEFAULT_IDLE_SLOT_LIMIT = 300; const char *msg_illregister = "Illegal op in register request received for job %s"; @@ -416,3 +417,22 @@ batch_request::batch_request(int type) : rq_type(type) { } + +int enqueue_threadpool_request( + + void *(*func)(void *), + void *arg, + threadpool_t *tp) + + { + return(0); + } + +threadpool_t *task_pool; + +void *svr_job_purge_task(void *vp) + { + return(NULL); + } + + diff --git a/src/test/req_register/test_uut.c b/src/test/req_register/test_uut.c index cd3611fac8..0974332900 100644 --- a/src/test/req_register/test_uut.c +++ b/src/test/req_register/test_uut.c @@ -697,7 +697,7 @@ START_TEST(set_depend_hold_test) struct depend *pdep; strcpy(pjob.ji_qs.ji_jobid, job1); - strcpy(pjob2.ji_qs.ji_jobid, job3); + strcpy(pjob2.ji_qs.ji_jobid, job2); pattr = &pjob.ji_wattr[JOB_ATR_depend]; initialize_depend_attr(pattr); // Job 2 will be complete with an exit status of 0 @@ -715,9 +715,17 @@ START_TEST(set_depend_hold_test) // Job 2 will be complete with an exit status of 0 make_dependjob(pdep, job2); - set_depend_hold(&pjob2, pattr, NULL); - fail_unless((pjob2.ji_wattr[JOB_ATR_hold].at_flags & ATR_VFLAG_SET) == 0); - fail_unless(pjob2.ji_qs.ji_state != JOB_STATE_HELD); + int saved_err = 0; + try + { + set_depend_hold(&pjob2, pattr, NULL); + } + catch (int err) + { + saved_err = err; + } + + fail_unless(saved_err == PBSE_BADDEPEND); memset(&pjob, 0, sizeof(pjob)); pattr = &pjob.ji_wattr[JOB_ATR_depend]; From 5fb8ff1feb6947988cd6e0ac2ed08f2ca191985a Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 10 Mar 2017 11:00:01 -0700 Subject: [PATCH 189/312] TRQ-3905. Move the contact listener error message to level 6 instead of unconditional. --- src/server/run_sched.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/server/run_sched.c b/src/server/run_sched.c index 8873fc1450..2cdecdf485 100644 --- a/src/server/run_sched.c +++ b/src/server/run_sched.c @@ -360,16 +360,18 @@ static int contact_listener( if (sock < 0) { /* FAILURE */ - - sprintf(tmpLine, "%s %d - port %d %s", - msg_listnr_nocall, - l_idx, - listener_conns[l_idx].port, - EMsg); + if (LOGLEVEL >= 6) + { + sprintf(tmpLine, "%s %d - port %d %s", + msg_listnr_nocall, + l_idx, + listener_conns[l_idx].port, + EMsg); + log_err(errno, __func__, tmpLine); + } /* we lost contact with the scheduler. reset*/ listener_conns[l_idx].first_time = 1; - log_err(errno, __func__, tmpLine); return(-1); } From 4527174cc8824ccb61aca7d88f99fbe63fa07469 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 13 Mar 2017 10:43:52 -0600 Subject: [PATCH 190/312] TRQ-3825. Validate the value for minwclimit to be a time string. --- src/include/attribute.h | 1 + src/lib/Libattr/attr_fn_resc.c | 11 ++- src/lib/Libattr/attr_fn_time.c | 139 ++++++++++++++++------------ src/server/resc_def_all.c | 34 ++++++- src/test/attr_fn_time/test_uut.c | 20 +++- src/test/job_recov/scaffolding.c | 5 + src/test/resc_def_all/scaffolding.c | 4 + src/test/svr_jobfunc/scaffolding.c | 5 + 8 files changed, 150 insertions(+), 69 deletions(-) diff --git a/src/include/attribute.h b/src/include/attribute.h index 734ea84c8b..8fbff163b8 100644 --- a/src/include/attribute.h +++ b/src/include/attribute.h @@ -383,6 +383,7 @@ void attr_atomic_kill(pbs_attribute *temp, attribute_def *pdef, int); void overwrite_complete_req(pbs_attribute *attr, pbs_attribute *new_attr); +long time_str_to_seconds(const char *str); int decode_b(pbs_attribute *patr, const char *name, const char *rn, const char *val, int); int decode_c(pbs_attribute *patr, const char *name, const char *rn, const char *val, int); int decode_l(pbs_attribute *patr, const char *name, const char *rn, const char *val, int); diff --git a/src/lib/Libattr/attr_fn_resc.c b/src/lib/Libattr/attr_fn_resc.c index bea63e4be3..3998a6e7cd 100644 --- a/src/lib/Libattr/attr_fn_resc.c +++ b/src/lib/Libattr/attr_fn_resc.c @@ -755,6 +755,8 @@ int action_resc( int actmode) { + int rc = PBSE_NONE; + if (pattr->at_val.at_ptr != NULL) { std::vector *resources = (std::vector *)pattr->at_val.at_ptr; @@ -765,12 +767,17 @@ int action_resc( if ((r.rs_value.at_flags & ATR_VFLAG_MODIFY) && (r.rs_defin->rs_action)) - r.rs_defin->rs_action(&r, pattr, actmode); + { + int tmp_rc = r.rs_defin->rs_action(&r, pattr, actmode); + if ((tmp_rc != PBSE_NONE) && + (rc == PBSE_NONE)) + rc = tmp_rc; + } resources->at(i).rs_value.at_flags &= ~ATR_VFLAG_MODIFY; } } - return(PBSE_NONE); + return(rc); } // END action_resc() diff --git a/src/lib/Libattr/attr_fn_time.c b/src/lib/Libattr/attr_fn_time.c index 261f6c8d25..733dbab76e 100644 --- a/src/lib/Libattr/attr_fn_time.c +++ b/src/lib/Libattr/attr_fn_time.c @@ -111,80 +111,61 @@ * value type "long" * -------------------------------------------------- */ +#define PBS_MAX_TIME (LONG_MAX - 1) + /* - * decode_time - decode time into into pbs_attribute structure of type ATR_TYPE_LONG + * time_str_to_seconds() * - * Returns: 0 if ok - * >0 error number if error - * *patr elements set + * Determines a number of seconds from a time string in the format: + * [DD:]HH:MM:SS[.MS] + * + * @param str - the time string + * @return the number of seconds or -1 if the format is invalid */ -#define PBS_MAX_TIME (LONG_MAX - 1) +long time_str_to_seconds( -int decode_time( - - pbs_attribute *patr, /* I/O (modified) */ - const char * UNUSED(name), /* I - pbs_attribute name (not used) */ - const char * UNUSED(rescn), /* I - resource name (not used) */ - const char *val, /* I - pbs_attribute value */ - int UNUSED(perm)) /* only used for resources */ + const char *str) { - int i; + long time_val = 0; + + char *workval = strdup(str); char msec[4]; + + char *workvalsv = workval; + bool use_days = false; int ncolon = 0; - int use_days = 0; int days = 0; - char *pc; - long rv = 0; - char *workval; - char *workvalsv; - - if ((val == NULL) || (strlen(val) == 0)) - { - patr->at_flags = (patr->at_flags & ~ATR_VFLAG_SET) | ATR_VFLAG_MODIFY; - - patr->at_val.at_long = 0; - - /* SUCCESS */ - - return(0); - } - - /* FORMAT: [DD]:HH:MM:SS[.MS] */ - - workval = strdup(val); - - workvalsv = workval; if (workvalsv == NULL) { /* FAILURE - cannot alloc memory */ - goto badval; + return(-1); } - for (i = 0;i < 3;++i) + for (int i = 0;i < 3;++i) msec[i] = '0'; - msec[i] = '\0'; + msec[4] = '\0'; - for (pc = workval;*pc;++pc) + for (char *pc = workval; *pc; ++pc) { if (*pc == ':') { if (++ncolon > 3) - goto badval; + return(-1); /* are days specified? */ if (ncolon > 2) - use_days = 1; + use_days = true; } } - for (pc = workval;*pc;++pc) + for (char *pc = workval; *pc; ++pc) { if (*pc == ':') { @@ -193,12 +174,12 @@ int decode_time( if (use_days) { - days = atoi(workval); - use_days = 0; + days = strtol(workval, NULL, 10); + use_days = false; } else { - rv = (rv * 60) + atoi(workval); + time_val = (time_val * 60) + strtol(workval, NULL, 10); } workval = pc + 1; @@ -208,43 +189,79 @@ int decode_time( { *pc++ = '\0'; - for (i = 0; (i < 3) && *pc; ++i) + for (int i = 0; (i < 3) && *pc; ++i) msec[i] = *pc++; break; } else if (!isdigit((int)*pc)) { - goto badval; /* bad value */ + return(-1); /* bad value */ } } - rv = (rv * 60) + atoi(workval); + time_val = (time_val * 60) + strtol(workval, NULL, 10); if (days > 0) - rv = rv + (days * 24 * 3600); + time_val = time_val + (days * 24 * 3600); - if (rv > PBS_MAX_TIME) - goto badval; + if (time_val > PBS_MAX_TIME) + return(-1); - if (atoi(msec) >= 500) - rv++; + if (strtol(msec, NULL, 10) >= 500) + time_val++; - patr->at_val.at_long = rv; + free(workvalsv); - patr->at_flags |= ATR_VFLAG_SET | ATR_VFLAG_MODIFY; + return(time_val); + } // END time_string_to_seconds() - free(workvalsv); - /* SUCCESS */ - return(0); +/* + * decode_time - decode time into into pbs_attribute structure of type ATR_TYPE_LONG + * + * Returns: 0 if ok + * >0 error number if error + * *patr elements set + */ -badval: - free(workvalsv); +int decode_time( + + pbs_attribute *patr, /* I/O (modified) */ + const char * UNUSED(name), /* I - pbs_attribute name (not used) */ + const char * UNUSED(rescn), /* I - resource name (not used) */ + const char *val, /* I - pbs_attribute value */ + int UNUSED(perm)) /* only used for resources */ + + { + long rv = 0; + int rc = PBSE_NONE; + + if ((val == NULL) || (strlen(val) == 0)) + { + patr->at_flags = (patr->at_flags & ~ATR_VFLAG_SET) | ATR_VFLAG_MODIFY; + + patr->at_val.at_long = 0; + + /* SUCCESS */ + + return(rc); + } + + rv = time_str_to_seconds(val); + + if (rv >= 0) + { + // SUCCESS + patr->at_val.at_long = rv; + patr->at_flags |= ATR_VFLAG_SET | ATR_VFLAG_MODIFY; + } + else + rc = PBSE_BADATVAL; - return(PBSE_BADATVAL); + return(rc); } /* END decode_time() */ diff --git a/src/server/resc_def_all.c b/src/server/resc_def_all.c index 1621947c6d..47fb8a16a5 100644 --- a/src/server/resc_def_all.c +++ b/src/server/resc_def_all.c @@ -123,7 +123,7 @@ int set_tokens_nodect(struct pbs_attribute *attr, struct pbs_attribute *new_attr int set_mppnodect(resource *, pbs_attribute *, int actmode); int decode_procct(pbs_attribute *, const char *, const char *, const char *, int); int encode_procct(pbs_attribute *, tlist_head *phead, const char *atname, const char *rsname, int mode, int perm); - +int verify_as_time(resource *r, pbs_attribute *pattr, int mode); resource_def *svr_resc_def; @@ -733,7 +733,7 @@ resource_def svr_resc_def_const[] = { "loglevel", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, { "minprocspeed", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, { "minpreempttime", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, - { "minwclimit", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, + { "minwclimit", decode_str, encode_str, set_str, comp_str, free_str, verify_as_time, READ_WRITE, ATR_TYPE_STR }, { "naccesspolicy", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, { "nallocpolicy", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, { "nodeset", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, @@ -1451,3 +1451,33 @@ int encode_procct( } + +/* + * verify_as_time() + * + * Verifies that the string set for this resource is a valid time string and can be + * converted to seconds. + * + * @param r - the resource in question (ignored) + * @param pattr - the attribute value for the resource + * @param mode - the mode of the action taken for this resource (ignored) + * @return PBSE_NONE on success, or -1 for failure. + */ + +int verify_as_time( + + resource *r, + pbs_attribute *pattr, + int mode) + + { + if (pattr->at_val.at_str != NULL) + { + if (time_str_to_seconds(pattr->at_val.at_str) < 0) + return(-1); + } + + return(PBSE_NONE); + } // END verify_as_time() + + diff --git a/src/test/attr_fn_time/test_uut.c b/src/test/attr_fn_time/test_uut.c index 1a46d84007..2ff0cf75d3 100644 --- a/src/test/attr_fn_time/test_uut.c +++ b/src/test/attr_fn_time/test_uut.c @@ -7,6 +7,7 @@ #include "pbs_error.h" int get_time_string(char *time_string, int string_size, long timeval); +long time_str_to_seconds(const char *str); START_TEST(test_get_time_string) { @@ -25,10 +26,21 @@ START_TEST(test_get_time_string) } END_TEST -START_TEST(test_two) +START_TEST(test_time_str_to_seconds) { + const char *s1 = "10:09:08:07.654"; // DD:HH:MM:SS.mmm (.milliseconds) + const char *s2 = "18:00:00"; // HH:MM::SS + const char *s3 = "bob"; // bad + const char *s4 = "18:00:00:00:00:00"; // bad + const char *s5 = "18:00:00:walltime=18:00:00"; // bad + const char *s6 = "00:04:00"; // HH:MM:SS - + fail_unless(time_str_to_seconds(s1) == 896888); // round up for .654 + fail_unless(time_str_to_seconds(s2) == 64800); + fail_unless(time_str_to_seconds(s3) == -1); + fail_unless(time_str_to_seconds(s4) == -1); + fail_unless(time_str_to_seconds(s5) == -1); + fail_unless(time_str_to_seconds(s6) == 240); } END_TEST @@ -39,8 +51,8 @@ Suite *attr_fn_time_suite(void) tcase_add_test(tc_core, test_get_time_string); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_time_str_to_seconds"); + tcase_add_test(tc_core, test_time_str_to_seconds); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/job_recov/scaffolding.c b/src/test/job_recov/scaffolding.c index 13f8ff80fc..b067a326c9 100644 --- a/src/test/job_recov/scaffolding.c +++ b/src/test/job_recov/scaffolding.c @@ -557,3 +557,8 @@ batch_request *get_remove_batch_request( { return(NULL); } + +long time_str_to_seconds(const char *str) + { + return(0); + } diff --git a/src/test/resc_def_all/scaffolding.c b/src/test/resc_def_all/scaffolding.c index 66062f4712..d30bf6b85a 100644 --- a/src/test/resc_def_all/scaffolding.c +++ b/src/test/resc_def_all/scaffolding.c @@ -282,3 +282,7 @@ char *threadsafe_tokenizer( return(start); } /* END threadsafe_tokenizer() */ +long time_str_to_seconds(const char *str) + { + return(0); + } diff --git a/src/test/svr_jobfunc/scaffolding.c b/src/test/svr_jobfunc/scaffolding.c index 80c1c1d932..2f106a0e36 100644 --- a/src/test/svr_jobfunc/scaffolding.c +++ b/src/test/svr_jobfunc/scaffolding.c @@ -742,3 +742,8 @@ job::~job() {} #include "../../lib/Libattr/req.cpp" #include "../../lib/Libattr/complete_req.cpp" #include "../../lib/Libattr/attr_req_info.cpp" + +long time_str_to_seconds(const char *str) + { + return(0); + } From 534e23cba6778373d6f0b292aa8306df2df94e72 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 13 Mar 2017 11:43:52 -0600 Subject: [PATCH 191/312] TRQ-3825. Fix two errors in the check-in. --- src/lib/Libattr/attr_fn_time.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/Libattr/attr_fn_time.c b/src/lib/Libattr/attr_fn_time.c index 733dbab76e..cb6d5ac124 100644 --- a/src/lib/Libattr/attr_fn_time.c +++ b/src/lib/Libattr/attr_fn_time.c @@ -150,14 +150,17 @@ long time_str_to_seconds( for (int i = 0;i < 3;++i) msec[i] = '0'; - msec[4] = '\0'; + msec[3] = '\0'; for (char *pc = workval; *pc; ++pc) { if (*pc == ':') { if (++ncolon > 3) + { + free(workvalsv); return(-1); + } /* are days specified? */ if (ncolon > 2) @@ -196,6 +199,7 @@ long time_str_to_seconds( } else if (!isdigit((int)*pc)) { + free(workvalsv); return(-1); /* bad value */ } } @@ -206,7 +210,10 @@ long time_str_to_seconds( time_val = time_val + (days * 24 * 3600); if (time_val > PBS_MAX_TIME) + { + free(workvalsv); return(-1); + } if (strtol(msec, NULL, 10) >= 500) time_val++; From f023237b47b2d1b3c0de4311836addeaee107087 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 13 Mar 2017 16:11:39 -0600 Subject: [PATCH 192/312] TRQ-3875. Inform the server for job prologue retries. Now, we exit with a specific error code for a failed prologue and log a message about it on the server so that it is easier to diagnose. --- src/include/pbs_job.h | 1 + src/resmom/start_exec.c | 20 ++++++++++++++++---- src/server/req_jobobit.c | 12 ++++++++++++ src/test/req_jobobit/scaffolding.c | 6 +++++- src/test/req_jobobit/test_uut.c | 9 ++++++++- 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 1b3a183f78..52e01e18f0 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -1287,6 +1287,7 @@ dir so that job can be restarted */ #define JOB_EXEC_OVERLIMIT_WT -11 /* job exceeded a walltime limit */ #define JOB_EXEC_OVERLIMIT_CPUT -12 /* job exceeded a cpu time limit */ #define JOB_EXEC_RETRY_CGROUP -13 /* couldn't create the job's cgroups */ +#define JOB_EXEC_RETRY_PROLOGUE -14 /* prologue failed */ extern void depend_clrrdy(job *); extern int depend_on_que(pbs_attribute *, void *, int); diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index 9ab6be842f..a0aad5f71f 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -3356,6 +3356,14 @@ void handle_reservation( +/* + * handle_prologs() + * + * Run relevant prologue scripts and notify the mom of a job failure if one of the prologues + * fail. + * + */ + void handle_prologs( job *pjob, @@ -3374,7 +3382,7 @@ void handle_prologs( if ((TJE->is_interactive == FALSE) && (rc != 1)) { - starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY, sjr); + starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY_PROLOGUE, sjr); } else { @@ -3393,7 +3401,7 @@ void handle_prologs( if ((TJE->is_interactive == FALSE) && (rc != 1)) { - starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY, sjr); + starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY_PROLOGUE, sjr); } else { @@ -3425,7 +3433,7 @@ void handle_prologs( if ((TJE->is_interactive == FALSE) && (rc != 1)) { - starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY, sjr); + starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY_PROLOGUE, sjr); } else { @@ -3445,7 +3453,11 @@ void handle_prologs( - +/* + * start_interactive_session() + * + * Opens the sockets to talk to the waiting qsub command + */ int start_interactive_session( diff --git a/src/server/req_jobobit.c b/src/server/req_jobobit.c index 72531ae8ee..a7d500471a 100644 --- a/src/server/req_jobobit.c +++ b/src/server/req_jobobit.c @@ -3307,6 +3307,18 @@ int update_substate_from_exit_status( set_job_comment(pjob, pbse_to_txt(PBSE_CGROUP_CREATE_FAIL)); // Fall through intentionally + + case JOB_EXEC_RETRY_PROLOGUE: + + if (exitstatus == JOB_EXEC_RETRY_PROLOGUE) + { + snprintf(log_buf, sizeof(log_buf), + "Job %s is being retried due to a failed prologue. See the syslog on the mother superior for more details", + pjob->ji_qs.ji_jobid); + log_err(-1, __func__, log_buf); + } + + // Fall through intentionally case JOB_EXEC_RETRY: diff --git a/src/test/req_jobobit/scaffolding.c b/src/test/req_jobobit/scaffolding.c index 6b39bf339d..0be4e1f413 100644 --- a/src/test/req_jobobit/scaffolding.c +++ b/src/test/req_jobobit/scaffolding.c @@ -63,6 +63,7 @@ bool purged = false; bool completed = false; bool exited = false; long disable_requeue = 0; +std::string log_err_buf; completed_jobs_map_class completed_jobs_map; @@ -329,7 +330,10 @@ void log_event(int eventtype, int objclass, const char *objname, const char *tex { } -void log_err(int error, const char *func_id, const char *msg) {} +void log_err(int error, const char *func_id, const char *msg) + { + log_err_buf = msg; + } void log_record(int eventtype, int objclass, const char *objname, const char *text) {} diff --git a/src/test/req_jobobit/test_uut.c b/src/test/req_jobobit/test_uut.c index caf29c7795..1ddc1ac215 100644 --- a/src/test/req_jobobit/test_uut.c +++ b/src/test/req_jobobit/test_uut.c @@ -61,6 +61,7 @@ extern long disable_requeue; extern int attr_count; extern int next_count; extern int called_account_jobend; +extern std::string log_err_buf; void init_server() @@ -670,7 +671,13 @@ START_TEST(update_substate_from_exit_status_test) fail_unless(update_substate_from_exit_status(pjob, &alreadymailed,"Some random message") == PBSE_NONE); fail_unless(pjob->ji_qs.ji_substate == JOB_SUBSTATE_RUNNING, "Shouldn't update substate when job is being deleted"); - + + pjob->ji_being_deleted = false; + pjob->ji_qs.ji_substate = JOB_SUBSTATE_RUNNING; + pjob->ji_qs.ji_un.ji_exect.ji_exitstat = JOB_EXEC_RETRY_PROLOGUE; + fail_unless(update_substate_from_exit_status(pjob, &alreadymailed,"Some random message") == PBSE_NONE); + fail_unless(pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN); + fail_unless(log_err_buf.find("prologue") != std::string::npos); } END_TEST From cdf2a9e559cd2e1e8b7065d3b4e9cb862746f3e9 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 14 Mar 2017 10:16:16 -0600 Subject: [PATCH 193/312] Fix qsub man page (-w option). --- doc/man1/qsub.1.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/man1/qsub.1.in b/doc/man1/qsub.1.in index ddb3728806..4248a438ce 100644 --- a/doc/man1/qsub.1.in +++ b/doc/man1/qsub.1.in @@ -87,7 +87,7 @@ qsub [\-a date_time] [\-A account_string] [\-b secs] [\-c checkpoint_options] [\-l resource_list] [\-m mail_options] [\-M user_list] [\-n node_exclusive] [\-N name] [\-o path] [\-p priority] [\-P proxy_username[:group]] [\-q destination] [\-r c] [\-S path_list] [\-t array_request] [\-T prologue/epilogue script_name] -[\-u user_list] [\-v variable_list] [\-V] [\-w] path +[\-u user_list] [\-v variable_list] [\-V] [\-w path] [\-W additional_attributes] [\-x] [\-X] [\-z] [script] .SH DESCRIPTION To create a job is to submit an executable script to a batch server. From 17ade676ba20ee7bebabde55363d42bac0c2ee99 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 14 Mar 2017 11:16:02 -0600 Subject: [PATCH 194/312] DOC-3098 - update man page for qsub to include -F and -L options. --- doc/man1/qsub.1.in | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/doc/man1/qsub.1.in b/doc/man1/qsub.1.in index 4248a438ce..955d291f4e 100644 --- a/doc/man1/qsub.1.in +++ b/doc/man1/qsub.1.in @@ -83,8 +83,8 @@ qsub \- submit pbs job .SH SYNOPSIS qsub [\-a date_time] [\-A account_string] [\-b secs] [\-c checkpoint_options] [\-C directive_prefix] -[\-d path] [\-D path] [\-e path] [\-f] [\-h] [\-i idle_slot_limit] [\-I] [\-j join] [\-k keep] [\-K kill_delay] -[\-l resource_list] [\-m mail_options] [\-M user_list] [\-n node_exclusive] [\-N name] +[\-d path] [\-D path] [\-e path] [\-f] [\-F] [\-h] [\-i idle_slot_limit] [\-I] [\-j join] [\-k keep] [\-K kill_delay] +[\-l resource_list] [\-L NUMA_resource_list] [\-m mail_options] [\-M user_list] [\-n node_exclusive] [\-N name] [\-o path] [\-p priority] [\-P proxy_username[:group]] [\-q destination] [\-r c] [\-S path_list] [\-t array_request] [\-T prologue/epilogue script_name] [\-u user_list] [\-v variable_list] [\-V] [\-w path] @@ -392,6 +392,13 @@ attribute will be set to true, which indicates that the job can survive the loss of a mom other than the "mother superior" mom (the first node in the exec hosts ) .Ig +.IP "\-F" 8 +Specifies the arguments that will be passed to the job script when the script is launched. +.br +The accepted syntax is: +.br +qsub -F "myarg1 myarg2 myarg3=myarg3value" myscript2.sh +.Ig .IP "\-h" 8 Specifies that a user hold be applied to the job at submission time. .if !\n(Pb .ig Ig @@ -535,6 +542,23 @@ Other than syntax, qsub performs no resource or value checking. The checking is performed by the execution server. .Ig .RE +.IP "\-L NUMA_resource_list" 8 +Defines the NUMA-aware resource requests for NUMA hardware. This option will work with non-NUMA hardware. +Syntax for +.Ar NUMA_resource_list +is: +.br +.Ty "\ \ \ \ tasks=#[:lprocs=#|all] +.br +.Ty "\ \ \ \ [:{usecores|usethreads|allowthreads}] +.br +.Ty "\ \ \ \ [:place={socket|numanode|core|thread}[=#]{node}][:memory=#][:swap=#][:maxtpn=#][:gpus=#[:]][:mics=#] +.br +.Ty "\ \ \ \ [:gres=][:feature=] +.br +.Ty "\ \ \ \ [[:{cpt|cgroup_per_task}]|[:{cph|cgroup_per_host}]] +.Ig +.RE .IP "\-m mail_options " 8 Defines the set of conditions under which the execution server will send a mail message about the job. The mail_options From 87fefe3a14e5fc3665588a9d6f76bbffd650ddd3 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 14 Mar 2017 13:48:55 -0600 Subject: [PATCH 195/312] Fix two crashes found in the regressions last night. --- src/server/process_alps_status.c | 4 +- src/server/process_mom_update.c | 109 ++++++++++++++++--------------- 2 files changed, 60 insertions(+), 53 deletions(-) diff --git a/src/server/process_alps_status.c b/src/server/process_alps_status.c index 316878370f..8415492a07 100644 --- a/src/server/process_alps_status.c +++ b/src/server/process_alps_status.c @@ -768,8 +768,8 @@ int process_alps_status( set_ncpus(current, parent, ncpus); #ifdef PENABLE_LINUX_CGROUPS - if (numa_nodes == 0) - numa_nodes = 1; + if (numa_nodes < sockets) + numa_nodes = sockets; if ((current->nd_layout.is_initialized() == false) || (current->nd_layout.getTotalThreads() != current->nd_slots.get_total_execution_slots())) diff --git a/src/server/process_mom_update.c b/src/server/process_mom_update.c index 15c19a7924..304fdc79dc 100644 --- a/src/server/process_mom_update.c +++ b/src/server/process_mom_update.c @@ -113,16 +113,16 @@ int gpu_has_job(struct pbsnode *pnode, int gpuid); void move_past_mic_status( - unsigned int &i, + unsigned int &status_index, std::vector &status_info) { - while (i < status_info.size()) + while (status_index < status_info.size()) { - if (!strcmp(status_info[i].c_str(), END_MIC_STATUS)) + if (!strcmp(status_info[status_index].c_str(), END_MIC_STATUS)) break; - i++; + status_index++; } } /* END move_past_mic_status() */ @@ -154,7 +154,7 @@ int save_single_mic_status( int process_mic_status( struct pbsnode *pnode, - unsigned int &i, + unsigned int &status_index, std::vector &status_info) { @@ -168,13 +168,13 @@ int process_mic_status( if ((rc = decode_arst(&temp, NULL, NULL, NULL, 0)) != PBSE_NONE) { log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, __func__, "cannot initialize attribute"); - move_past_mic_status(i, status_info); + move_past_mic_status(status_index, status_info); return(rc); } - for (i++; i < status_info.size(); i++) + for (status_index++; status_index < status_info.size(); status_index++) { - const char *str = status_info[i].c_str(); + const char *str = status_info[status_index].c_str(); if (!strcmp(str, END_MIC_STATUS)) break; @@ -214,7 +214,7 @@ int process_mic_status( } } - move_past_mic_status(i, status_info); + move_past_mic_status(status_index, status_info); node_micstatus_list(&temp, pnode, ATR_ACTION_ALTER); @@ -915,16 +915,19 @@ int process_status_info( else if (!strcmp(str, START_GPU_STATUS)) { is_gpustat_get(current, i, status_info); - str = status_info[i].c_str(); + continue; } else if (!strcmp(str, START_MIC_STATUS)) { process_mic_status(current, i, status_info); - str = status_info[i].c_str(); + continue; } #ifdef PENABLE_LINUX_CGROUPS else if (!strcmp(str, "force_layout_update")) + { force_layout_update = true; + continue; + } else if (!strncmp(str, "layout", 6)) { // Add 7 to skip "layout=" @@ -962,9 +965,12 @@ int process_status_info( /* reset gpu data in case mom reconnects with changed gpus */ clear_nvidia_gpus(current); + + continue; } else { + // Save this string to our status line. if (temp.size() > 0) temp += ","; @@ -983,52 +989,53 @@ int process_status_info( } else temp += str; - } - - if (!strncmp(str, "state", 5)) - { - if (dont_change_state == FALSE) - process_state_str(current, str); - } - else if ((allow_any_mom == TRUE) && - (!strncmp(str, "uname", 5))) - { - process_uname_str(current, str); - } - else if (!strncmp(str, "me", 2)) /* shorter str compare than "message" */ - { - if ((!strncmp(str, "message=ERROR", 13)) && - (down_on_error == TRUE)) + + if (!strncmp(str, "state", 5)) { - update_node_state(current, INUSE_DOWN); - dont_change_state = TRUE; - - if (note_append_on_error == true) + if (dont_change_state == FALSE) + process_state_str(current, str); + } + else if ((allow_any_mom == TRUE) && + (!strncmp(str, "uname", 5))) + { + process_uname_str(current, str); + } + else if (!strncmp(str, "me", 2)) /* shorter str compare than "message" */ + { + if ((!strncmp(str, "message=ERROR", 13)) && + (down_on_error == TRUE)) { - set_note_error(current, str); + update_node_state(current, INUSE_DOWN); + dont_change_state = TRUE; + + if (note_append_on_error == true) + { + set_note_error(current, str); + } } } - } - else if (!strncmp(str,"macaddr=",8)) - { - update_node_mac_addr(current,str + 8); - } - else if ((mom_job_sync == true) && - (!strncmp(str, "jobdata=", 8))) - { - /* update job attributes based on what the MOM gives us */ - update_job_data(current, str + strlen("jobdata=")); - } - else if ((auto_np) && - (!(strncmp(str, "ncpus=", 6)))) + else if (!strncmp(str,"macaddr=",8)) + { + update_node_mac_addr(current,str + 8); + } + else if ((mom_job_sync == true) && + (!strncmp(str, "jobdata=", 8))) + { + /* update job attributes based on what the MOM gives us */ + update_job_data(current, str + strlen("jobdata=")); + } + else if ((auto_np) && + (!(strncmp(str, "ncpus=", 6)))) - { - handle_auto_np(current, str); - } - else if (!strncmp(str, "version=", 8)) - { - current->set_version(str + 8); + { + handle_auto_np(current, str); + } + else if (!strncmp(str, "version=", 8)) + { + current->set_version(str + 8); + } } + } /* END processing strings */ if (current != NULL) From 6138d9a624e3a0fd70e8648ea98c414e11615cd0 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 15 Mar 2017 14:59:16 -0600 Subject: [PATCH 196/312] TRQ-3913 - set correct gpu default when using CUDA 8 --- configure.ac | 1 + src/resmom/nvidia.c | 55 +++++++++++++- src/test/Makefile.am | 4 + src/test/node_manager/test_uut.c | 28 ------- src/test/nvidia/Makefile.am | 4 + src/test/nvidia/scaffolding.c | 34 +++++++++ src/test/nvidia/test_nvidia.h | 9 +++ src/test/nvidia/test_uut.c | 73 +++++++++++++++++++ src/test/process_mom_update/scaffolding.c | 2 +- .../receive_mom_communication/scaffolding.c | 1 + 10 files changed, 178 insertions(+), 33 deletions(-) create mode 100644 src/test/nvidia/Makefile.am create mode 100644 src/test/nvidia/scaffolding.c create mode 100644 src/test/nvidia/test_nvidia.h create mode 100644 src/test/nvidia/test_uut.c diff --git a/configure.ac b/configure.ac index d83745552e..d4a55836cc 100644 --- a/configure.ac +++ b/configure.ac @@ -494,6 +494,7 @@ m4_ifdef([AM_SILENT_RULES],[ src/test/mom_server/Makefile src/test/mom_process_request/Makefile src/test/mom_req_quejob/Makefile + src/test/nvidia/Makefile src/test/parse_config/Makefile src/test/pbs_demux/Makefile src/test/pmix_interface/Makefile diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index 527edeec20..3a0c630919 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -648,6 +648,31 @@ static int gpumodes( return(TRUE); } +/* + * Return the NVML API version + * + * Assumptions: Library has already been initialized (nvmlInit() has been called). + */ + +int get_nvml_version() + { + static int version = -1; + +#ifdef NVML_API + if (version == -1) + { + char version_buf[NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE]; + + // the nvml version is returned as a string in the form x.y.z + // only return major version (x) + if (nvmlSystemGetNVMLVersion(version_buf, sizeof(version_buf)) == NVML_SUCCESS) + version = atoi(version_buf); + } +#endif + + return(version); + } + /* * Function to set gpu mode @@ -687,7 +712,19 @@ int setgpumode( case gpu_exclusive_thread: - compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_THREAD; + if (get_nvml_version() >= 8) + { + sprintf(log_buffer, "exclusive thread compute mode is not allowed in NVML version %d. Setting exclusive process compute mode instead.", + get_nvml_version()); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + + compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_PROCESS; + } + else + { + compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_THREAD; + } + break; case gpu_prohibited: @@ -773,6 +810,16 @@ int setgpumode( } else /* 270 or greater driver */ { + // exclusive thread mode no longer allowed starting with driver version 367 + if ((MOMNvidiaDriverVersion >= 367) && (gpumode == gpu_exclusive_thread)) + { + sprintf(log_buffer, "exclusive thread compute mode is not allowed with NVIDIA driver version %d. Setting exclusive process compute mode instead.", + MOMNvidiaDriverVersion); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + + gpumode = gpu_exclusive_process; + } + sprintf(buf, "nvidia-smi -i %d -c %d 2>&1", gpuid, gpumode); @@ -784,8 +831,8 @@ int setgpumode( log_ext(-1, __func__, log_buffer, LOG_DEBUG); } - if ((fd = popen(buf, "r")) != NULL) - { + if ((fd = popen(buf, "r")) != NULL) + { while (!feof(fd)) { if (fgets(buf, 300, fd)) @@ -818,7 +865,7 @@ int setgpumode( } } pclose(fd); - } + } else { if (LOGLEVEL >= 0) diff --git a/src/test/Makefile.am b/src/test/Makefile.am index f931459ac5..52194066e6 100644 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -84,6 +84,10 @@ if BUILD_LINUX_CGROUPS MOM_UT_DIRS += trq_cgroups endif +if NVIDIA + MOM_UT_DIRS += nvidia +endif + if INCLUDE_PAM PAM_DIRS = pam endif diff --git a/src/test/node_manager/test_uut.c b/src/test/node_manager/test_uut.c index d9a2441890..89f62aef1f 100644 --- a/src/test/node_manager/test_uut.c +++ b/src/test/node_manager/test_uut.c @@ -14,9 +14,6 @@ #include "json/json.h" #include "complete_req.hpp" #include "pbs_nodes.h" -#ifdef NVML_API -#include -#endif const char *exec_hosts = "napali/0+napali/1+napali/2+napali/50+napali/4+l11/0+l11/1+l11/2+l11/3"; char buf[4096]; @@ -952,25 +949,6 @@ START_TEST(place_subnodes_in_hostlist_job_exclusive_test) END_TEST -#ifdef NVML_API -START_TEST(test_proplist) - { - const char *str[] = {"gpus=1"}; - std::vector plist; - int node_req; - int gpu_req; - int mic_req; - - fail_unless(proplist((char **)str, plist, &node_req, &gpu_req, &mic_req) == PBSE_NONE); -#if defined(NVML_API_VERSION) && (NVML_API_VERSION >= 8) - fail_unless(gpu_mode_rqstd == gpu_exclusive_process); -#else - fail_unless(gpu_mode_rqstd == gpu_exclusive_thread); -#endif - } -END_TEST -#endif - Suite *node_manager_suite(void) { Suite *s = suite_create("node_manager_suite methods"); @@ -1033,12 +1011,6 @@ Suite *node_manager_suite(void) tcase_add_test(tc_core, check_node_jobs_exitence_test); suite_add_tcase(s, tc_core); -#ifdef NVML_API - tc_core = tcase_create("test_proplist"); - tcase_add_test(tc_core, test_proplist); - suite_add_tcase(s, tc_core); -#endif - return(s); } diff --git a/src/test/nvidia/Makefile.am b/src/test/nvidia/Makefile.am new file mode 100644 index 0000000000..1827b7bc0a --- /dev/null +++ b/src/test/nvidia/Makefile.am @@ -0,0 +1,4 @@ + +include ../Makefile_Mom.ut + +libuut_la_SOURCES = ${PROG_ROOT}/nvidia.c diff --git a/src/test/nvidia/scaffolding.c b/src/test/nvidia/scaffolding.c new file mode 100644 index 0000000000..45ecc5d139 --- /dev/null +++ b/src/test/nvidia/scaffolding.c @@ -0,0 +1,34 @@ +#include "license_pbs.h" /* See here for the software license */ +#include +#include /* fprintf */ +#include /* winsize */ +#include /* termios */ +#include +#include "log.h" +#include "mcom.h" + +char log_buffer[LOG_BUF_SIZE]; + +int MXMLFromString( + + mxml_t **EP, /* O (populate or create) */ + char *XMLString, /* I */ + char **Tail, /* O (optional) */ + char *EMsg, /* O (optional) */ + int emsg_size) /* I */ + + { + return(0); + } + +int MXMLDestroyE(mxml_t **EP) {return(0); } + +void log_err(int errnum, const char *routine, const char *text) {} + +void log_ext(int errnum, const char *routine, const char *text, int severity) { } + +void log_event(int eventtype, int objclass, const char *objname, const char *text) {} + +void send_update_soon() { return; } + +void get_device_indices(const char *device_str, std::vector &device_indices, const char *suffix) {} diff --git a/src/test/nvidia/test_nvidia.h b/src/test/nvidia/test_nvidia.h new file mode 100644 index 0000000000..f9b8d20f3c --- /dev/null +++ b/src/test/nvidia/test_nvidia.h @@ -0,0 +1,9 @@ +#include "license_pbs.h" /* See here for the software license */ +#ifndef _NVIDIA_CT_H +#define _NVIDIA_CT_H +#include + +#define NVIDIA_SUITE 1 +Suite *nvidia_suite(); + +#endif /* _NVIDIA_CT_H */ diff --git a/src/test/nvidia/test_uut.c b/src/test/nvidia/test_uut.c new file mode 100644 index 0000000000..e6fff33605 --- /dev/null +++ b/src/test/nvidia/test_uut.c @@ -0,0 +1,73 @@ +#include "license_pbs.h" /* See here for the software license */ +#include "mom_mach.h" +#include "test_nvidia.h" +#include +#include +#include +#include +#include + +#include "pbs_error.h" +#include "pbs_job.h" + +int MOMNvidiaDriverVersion = 0; +int LOGLEVEL = 0; +int use_nvidia_gpu = TRUE; +int global_gpu_count = 0; +int time_now = 0; +char mom_host[PBS_MAXHOSTNAME + 1]; + +int get_nvml_version(); + +START_TEST(test_get_nvml_version) + { + int version; + + nvmlInit(); + + version = get_nvml_version(); + fail_unless(version == NVML_API_VERSION); + + nvmlShutdown(); + } +END_TEST + +START_TEST(test_two) + { + } +END_TEST + +Suite *nvidia_suite(void) + { + Suite *s = suite_create("nvidia_suite methods"); + TCase *tc_core = tcase_create("test_get_nvml_version"); + + // give this test a bit of extra time since nvmlInit() can take awhile + tcase_set_timeout(tc_core, 15); + + tcase_add_test(tc_core, test_get_nvml_version); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_two"); + tcase_add_test(tc_core, test_two); + suite_add_tcase(s, tc_core); + + return s; + } + +void rundebug() + { + } + +int main(void) + { + int number_failed = 0; + SRunner *sr = NULL; + rundebug(); + sr = srunner_create(nvidia_suite()); + srunner_set_log(sr, "nvidia_suite.log"); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return number_failed; + } diff --git a/src/test/process_mom_update/scaffolding.c b/src/test/process_mom_update/scaffolding.c index ca7d614503..aa360460f6 100644 --- a/src/test/process_mom_update/scaffolding.c +++ b/src/test/process_mom_update/scaffolding.c @@ -14,7 +14,7 @@ #include "u_tree.h" #include "id_map.hpp" -#include "id_map.hpp" +#include "machine.hpp" char server_name[PBS_MAXSERVERNAME + 1]; /* host_name[:service|port] */ int allow_any_mom; diff --git a/src/test/receive_mom_communication/scaffolding.c b/src/test/receive_mom_communication/scaffolding.c index 8d7caa21ca..a19648cdb9 100644 --- a/src/test/receive_mom_communication/scaffolding.c +++ b/src/test/receive_mom_communication/scaffolding.c @@ -17,6 +17,7 @@ #include "node_manager.h" #include "pbs_ifl.h" #include "authorized_hosts.hpp" +#include "machine.hpp" id_map job_mapper; From 9ab04c3f8be680416bcecab83a2056f63fc13449 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 16 Mar 2017 13:56:34 -0600 Subject: [PATCH 197/312] TRQ-3924 - fix pbs_server.in file --- contrib/init.d/pbs_server.in | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/contrib/init.d/pbs_server.in b/contrib/init.d/pbs_server.in index 4c27357b75..4a4ae63852 100644 --- a/contrib/init.d/pbs_server.in +++ b/contrib/init.d/pbs_server.in @@ -48,28 +48,29 @@ kill_pbs_server() { pid=`pidof_pbs_server` if [ $? -ne 0 ]; then - return 0; + return 0 fi - # Use a qterm that only acts locally - qterm -l + # Use a qterm that only acts locally + $BIN_PATH/qterm -l - # If our local qterm couldn't do anything, then send the signal as a last resort - if [ $? -ne 0]; then - kill -TERM $pid - fi + # If our local qterm couldn't do anything, then send the signal as a last resort + if [ $? -ne 0 ]; then + kill -TERM $pid + fi # wait for process to end for i in {1..5}; do - kill -0 $pid &>/dev/null || return 0 - sleep 1 + kill -0 $pid &>/dev/null || return 0 + sleep 1 done # waited too long, try again - qterm -l - if [ $? -ne 0]; then - killproc pbs_server -TERM - fi + $BIN_PATH/qterm -l + if [ $? -ne 0 ]; then + killproc pbs_server -TERM + fi + return $? } From 2729a515005fe9bcd576773863016be2add9948f Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 17 Mar 2017 16:59:01 -0600 Subject: [PATCH 198/312] TRQ-3965. Add diagnostics and correct the source of the gpu indices for cgroups. --- src/include/allocation.hpp | 17 ++++ src/include/machine.hpp | 3 +- src/include/pbs_ifl.h | 2 + src/include/pbs_job.h | 2 + src/lib/Libutils/allocation.cpp | 56 ++++++++++++- src/lib/Libutils/machine.cpp | 75 ++++++++---------- src/lib/Libutils/numa_chip.cpp | 7 +- src/resmom/start_exec.c | 24 ++++-- src/resmom/trq_cgroups.c | 64 ++++++++------- src/server/job_attr_def.c | 26 ++++++ src/server/node_manager.c | 119 ++++++++++++++++------------ src/test/job_func/test_uut.c | 3 +- src/test/machine/scaffolding.c | 2 +- src/test/machine/test_uut.c | 29 +++---- src/test/node_manager/scaffolding.c | 9 +-- src/test/node_manager/test_uut.c | 19 +++-- 16 files changed, 291 insertions(+), 166 deletions(-) diff --git a/src/include/allocation.hpp b/src/include/allocation.hpp index 5dc7b82b55..09be6ff473 100644 --- a/src/include/allocation.hpp +++ b/src/include/allocation.hpp @@ -87,6 +87,8 @@ extern const int MEM_INDICES; extern const int CPU_INDICES; +extern const int GPU_INDICES; +extern const int MIC_INDICES; extern const int exclusive_node; extern const int exclusive_socket; @@ -97,6 +99,20 @@ extern const int exclusive_legacy; extern const int exclusive_legacy2; extern const int exclusive_none; + +class cgroup_info + { + public: + + std::string mem_string; + std::string cpu_string; + std::string gpu_string; + std::string mic_string; + + cgroup_info() : mem_string(), cpu_string(), gpu_string(), mic_string() {} + }; + + // forward declare req class req; @@ -133,6 +149,7 @@ class allocation void set_cput_used(const unsigned long cput_used); void set_memory_used(const unsigned long long mem_used); void place_indices_in_string(std::string &output, int which); + void place_indices_in_string(cgroup_info &output); void set_place_type(const std::string &place); void get_place_type(int &place_type); void write_task_information(std::string &task_info) const; diff --git a/src/include/machine.hpp b/src/include/machine.hpp index 6383bb767d..289e9bac3a 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -353,13 +353,12 @@ class Machine void place_all_execution_slots(req &r, allocation &master, const char *hostname); int spread_place(req &r, allocation &master, int tasks_for_node, const char *hostname); int spread_place_pu(req &r, allocation &master, int tasks_for_node, const char *hostname); - int place_job(job *pjob, string &cpu_string, string &mem_string, const char *hostname, bool legacy_vmem); + int place_job(job *pjob, cgroup_info &cgi, const char *hostname, bool legacy_vmem); void setMemory(long long mem); void addSocket(int count); // used for unit tests void setIsNuma(bool is_numa); // used for unit tests void save_allocations(const Machine &other); void free_job_allocation(const char *jobid); - int get_jobs_cpusets(const char *jobid, string &cpus, string &mems); int fit_tasks_within_sockets(req &r, allocation &job_alloc, const char *hostname, int &remaining_tasks); void place_remaining(req &to_split, allocation &master, int &remaining_tasks, const char *hostname); int how_many_tasks_can_be_placed(req &r) const; diff --git a/src/include/pbs_ifl.h b/src/include/pbs_ifl.h index 89eb68cdfd..2c772b792e 100644 --- a/src/include/pbs_ifl.h +++ b/src/include/pbs_ifl.h @@ -381,6 +381,8 @@ #define ATTR_cpustr "cpuset_string" #define ATTR_memstr "memset_string" #define ATTR_L_request "L_Request" +#define ATTR_gpus_reserved "gpus_reserved" +#define ATTR_mics_reserved "mics_reserved" #define ATTR_user_kill_delay "user_kill_delay" #define ATTR_idle_slot_limit "idle_slot_limit" #define ATTR_default_gpu_mode "default_gpu_mode" diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 52e01e18f0..98a33aa1e9 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -437,6 +437,8 @@ enum job_atr JOB_ATR_user_kill_delay, JOB_ATR_idle_slot_limit, JOB_ATR_LRequest, + JOB_ATR_gpus_reserved, + JOB_ATR_mics_reserved, JOB_ATR_UNKN, /* the special "unknown" type */ JOB_ATR_LAST /* This MUST be LAST */ }; diff --git a/src/lib/Libutils/allocation.cpp b/src/lib/Libutils/allocation.cpp index eca35b0583..15e8bd591c 100644 --- a/src/lib/Libutils/allocation.cpp +++ b/src/lib/Libutils/allocation.cpp @@ -88,6 +88,8 @@ const int MEM_INDICES = 0; const int CPU_INDICES = 1; +const int GPU_INDICES = 2; +const int MIC_INDICES = 3; const int exclusive_none = 0; const int exclusive_node = 1; @@ -233,10 +235,41 @@ void allocation::place_indices_in_string( int which) { - if (which == MEM_INDICES) - translate_vector_to_range_string(output, this->mem_indices); - else - translate_vector_to_range_string(output, this->cpu_indices); + switch (which) + { + case MEM_INDICES: + + translate_vector_to_range_string(output, this->mem_indices); + break; + + case CPU_INDICES: + + translate_vector_to_range_string(output, this->cpu_indices); + break; + + case GPU_INDICES: + + translate_vector_to_range_string(output, this->gpu_indices); + break; + + case MIC_INDICES: + + translate_vector_to_range_string(output, this->mic_indices); + break; + } + } // END place_indices_in_string() + + + +void allocation::place_indices_in_string( + + cgroup_info &cgi) + + { + this->place_indices_in_string(cgi.mem_string, MEM_INDICES); + this->place_indices_in_string(cgi.cpu_string, CPU_INDICES); + this->place_indices_in_string(cgi.gpu_string, GPU_INDICES); + this->place_indices_in_string(cgi.mic_string, MIC_INDICES); } // END place_indices_in_string() @@ -289,12 +322,27 @@ void allocation::write_task_information( { std::string cpus; std::string mems; + std::string gpus; + std::string mics; char buf[256]; translate_vector_to_range_string(cpus, this->cpu_indices); translate_vector_to_range_string(mems, this->mem_indices); task_info = "{\"task\":{\"cpu_list\":\"" + cpus; task_info += "\",\"mem_list\":\"" + mems; + + if (this->gpu_indices.size() > 0) + { + translate_vector_to_range_string(gpus, this->gpu_indices); + task_info += "\",\"gpu_list\":\"" + gpus; + } + + if (this->mic_indices.size() > 0) + { + translate_vector_to_range_string(mics, this->mic_indices); + task_info += "\",\"mic_list\":\"" + mics; + } + if (this->task_cput_used != 0) { unsigned long cput_used = this->task_cput_used; diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index 94ba768906..ec1df4d32a 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -2,7 +2,9 @@ #include #include #include + #include "pbs_config.h" +#include "log.h" #ifdef PENABLE_LINUX_CGROUPS #include @@ -30,7 +32,6 @@ using namespace std; const int ALL_TASKS = -1; - /* 26 August 2014 * Currently Intel and AMD NUMA hardware * put their NUMA node and Socket in a different order. @@ -863,6 +864,7 @@ int Machine::spread_place( bool chips = false; int quantity = r.get_sockets(); int tasks_placed = 0; + char log_buf[LOCAL_LOG_BUF_SIZE]; if (quantity == 0) { @@ -906,6 +908,11 @@ int Machine::spread_place( // If we don't have enough memory, reject the job if (mem_needed > mem_count) { + snprintf(log_buf, sizeof(log_buf), + "Job %s requires at least %lukb memory from host %s, but only %lukb is available", + master.jobid.c_str(), mem_needed, hostname, mem_count); + log_err(-1, __func__, log_buf); + return(PBSE_RESCUNAV); } } @@ -953,7 +960,14 @@ int Machine::spread_place( } if (tasks_placed != tasks_for_node) + { + snprintf(log_buf, sizeof(log_buf), + "Job %s needed to place %d tasks on node %s, but could only place %d", + master.jobid.c_str(), tasks_for_node, hostname, tasks_placed); + log_err(-1, __func__, log_buf); + return(PBSE_IVALREQ); + } return(PBSE_NONE); } // END spread_place() @@ -997,11 +1011,10 @@ int Machine::fit_tasks_within_sockets( int Machine::place_job( - job *pjob, - string &cpu_string, - string &mem_string, - const char *hostname, - bool legacy_vmem) + job *pjob, + cgroup_info &cgi, + const char *hostname, + bool legacy_vmem) { int rc = PBSE_NONE; @@ -1011,6 +1024,7 @@ int Machine::place_job( vector partially_place; allocation job_alloc(pjob->ji_qs.ji_jobid); vector to_split; + char log_buf[LOCAL_LOG_BUF_SIZE]; // See if the tasks fit completely on a socket, and if they do then place them there for (int i = 0; i < num_reqs; i++) @@ -1049,7 +1063,14 @@ int Machine::place_job( // Placing 0 tasks is an error if (this->sockets[j].place_task(r, job_alloc, tasks_for_node, hostname) == 0) + { + snprintf(log_buf, sizeof(log_buf), + "Socket %u for node %s was thought to fit at least %d tasks, but couldn't fit any", + j, hostname, tasks_for_node); + log_err(-1, __func__, log_buf); + return(-1); + } break; } @@ -1089,12 +1110,16 @@ int Machine::place_job( if (remaining_tasks > 0) { // this allocation so that the cleanup happens correctly + snprintf(log_buf, sizeof(log_buf), + "Job %s could not place %d tasks from req %d on host %s", + pjob->ji_qs.ji_jobid, remaining_tasks, partially_place[i], hostname); + log_err(-1, __func__, log_buf); + rc = -1; } } - job_alloc.place_indices_in_string(mem_string, MEM_INDICES); - job_alloc.place_indices_in_string(cpu_string, CPU_INDICES); + job_alloc.place_indices_in_string(cgi); this->allocations.push_back(job_alloc); @@ -1103,40 +1128,6 @@ int Machine::place_job( -/* - * get_jobs_cpusets() - * - * Gets the cpusets corresponding to this job_id so that the child can place them - * @param job_id (I) - the id of the job - * @param cpus (O) - put the string representing the cpus here - * @param mems (O) - put the string representing the memory nodes here - * @return PBSE_NONE if we found the job, -1 otherwise - */ - -int Machine::get_jobs_cpusets( - - const char *job_id, - string &cpus, - string &mems) - - { - int rc = -1; - - for (unsigned int i = 0; i < this->allocations.size(); i++) - { - if (this->allocations[i].jobid == job_id) - { - this->allocations[i].place_indices_in_string(mems, MEM_INDICES); - this->allocations[i].place_indices_in_string(cpus, CPU_INDICES); - rc = PBSE_NONE; - } - } - - return(rc); - } // END get_jobs_cpusets() - - - void Machine::free_job_allocation( const char *job_id) diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index 79d285f697..a76ae92954 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -1740,7 +1740,8 @@ bool Chip::spread_place_threads( return(placed); - } + } // END spread_place_threads() + /* spread_place_cores @@ -1863,7 +1864,7 @@ bool Chip::spread_place_cores( return(placed); - } + } // END spread_place_cores() @@ -2048,7 +2049,7 @@ int Chip::place_task( else { int threads_to_rsv = execution_slots_per_task; - if(r.getPlaceThreads() > 0) + if (r.getPlaceThreads() > 0) threads_to_rsv = r.getPlaceThreads(); place_tasks_execution_slots(execution_slots_per_task, threads_to_rsv, task_alloc, THREAD_INT); diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index a0aad5f71f..0be6cfac72 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -3236,6 +3236,9 @@ void handle_cpuset_creation( +/* + * handle_reservation() + */ void handle_reservation( @@ -3595,6 +3598,9 @@ int start_interactive_session( +/* + * start_interactive_reader() + */ void start_interactive_reader( @@ -3625,6 +3631,9 @@ void start_interactive_reader( +/* + * setup_interacteractive_job() + */ void setup_interactive_job( @@ -4752,15 +4761,16 @@ int TMomFinalizeChild( // Create the cgroups for this job rc = init_torque_cgroups(); - if (rc == PBSE_NONE) { - rc = trq_cg_create_all_cgroups(pjob); - } else { - /* - Send/log an additional error, cleanup and return as before - */ + if (rc == PBSE_NONE) + { + rc = trq_cg_create_all_cgroups(pjob); + } + else + { + // Log an additional error, cleanup and return as before sprintf(log_buffer, "Could not init cgroups for job %s.", pjob->ji_qs.ji_jobid); log_ext(-1, __func__, log_buffer, LOG_ERR); - } + } if (rc != PBSE_NONE) { diff --git a/src/resmom/trq_cgroups.c b/src/resmom/trq_cgroups.c index 6c2dffeb1b..d17cc3475f 100644 --- a/src/resmom/trq_cgroups.c +++ b/src/resmom/trq_cgroups.c @@ -2090,6 +2090,7 @@ int trq_cg_set_forbidden_devices(std::vector &forbidden_devices, std::strin } + /* * trq_cg_add_devices_to_cgroups * @@ -2106,26 +2107,17 @@ int trq_cg_add_devices_to_cgroup( job *pjob) { - std::vector device_indices; + std::vector device_indices; std::vector forbidden_devices; std::string job_devices_path; char log_buf[LOCAL_LOG_BUF_SIZE]; int rc = PBSE_NONE; - char *device_str; - char suffix[20]; unsigned int device_count = 0; - int index = 0; #ifdef NVIDIA_GPUS device_count = global_gpu_count; - strcpy(suffix, "-gpu"); - index = JOB_ATR_exec_gpus; -#endif - -#ifdef MIC +#elif defined(MIC) device_count = global_mic_count; - strcpy(suffix, "-mic"); - index = JOB_ATR_exec_mics; #endif /* First make sure we have gpus */ @@ -2134,10 +2126,15 @@ int trq_cg_add_devices_to_cgroup( job_devices_path = cg_devices_path + pjob->ji_qs.ji_jobid; - - /* if there are no gpus given, deny all gpus to the job */ - if (((pjob->ji_wattr[index].at_flags & ATR_VFLAG_SET) == 0) || - (pjob->ji_wattr[index].at_val.at_str == NULL)) +#ifdef NVIDIA_GPUS + if (pjob->ji_wattr[JOB_ATR_gpus_reserved].at_val.at_str != NULL) + { + std::string gpus_reserved(pjob->ji_wattr[JOB_ATR_gpus_reserved].at_val.at_str); + std::string gpu_range; + find_range_in_cpuset_string(gpus_reserved, gpu_range); + translate_range_string_to_vector(gpu_range.c_str(), device_indices); + } + else { for (unsigned int i = 0; i < device_count; i++) forbidden_devices.push_back(i); @@ -2149,16 +2146,29 @@ int trq_cg_add_devices_to_cgroup( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); return(rc); } - - return(rc); } +#elif defined(MIC) + if (pjob->ji_wattr[JOB_ATR_mics_reserved].at_val.at_str != NULL) + { + std::string mics_reserved(pjob->ji_wattr[JOB_ATR_mics_reserved].at_val.at_str); + std::string mic_range; + find_range_in_cpuset_string(mics_reserved, mic_range); + translate_range_string_to_vector(mic_range.c_str(), device_indices); + } + else + { + for (unsigned int i = 0; i < device_count; i++) + forbidden_devices.push_back(i); - - device_str = pjob->ji_wattr[index].at_val.at_str; - - device_indices.clear(); - get_device_indices(device_str, device_indices, suffix); - + rc = trq_cg_set_forbidden_devices(forbidden_devices, job_devices_path); + if (rc != PBSE_NONE) + { + sprintf(log_buf, "Failed to write devices.deny list for job %s", pjob->ji_qs.ji_jobid); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); + return(rc); + } + } +#endif /* device_indices has the list of gpus for this job. make a list of gpus that are not for this job @@ -2168,17 +2178,17 @@ int trq_cg_add_devices_to_cgroup( bool found; found = false; - for(std::vector::iterator it = device_indices.begin(); it != device_indices.end(); it++) + for (size_t j = 0; j < device_indices.size(); j++) { - if (*it == i) + if (device_indices[j] == i) { found = true; break; } } + if (found == false) forbidden_devices.push_back(i); - } if (forbidden_devices.size() == 0) @@ -2193,6 +2203,6 @@ int trq_cg_add_devices_to_cgroup( } return(rc); - } + } // END trq_cg_add_devices_to_cgroup() #endif diff --git a/src/server/job_attr_def.c b/src/server/job_attr_def.c index 495767ec8d..fd175ff844 100644 --- a/src/server/job_attr_def.c +++ b/src/server/job_attr_def.c @@ -1331,6 +1331,32 @@ attribute_def job_attr_def[] = ATR_TYPE_STR, PARENT_TYPE_JOB, }, + + // JOB_ATR_gpus_reserved + {(char *)ATTR_gpus_reserved, /* "gpus_reserved" */ + decode_str, + encode_str, + set_str, + comp_str, + free_str, + NULL_FUNC, + READ_ONLY | ATR_DFLAG_MOM | ATR_DFLAG_OPWR | ATR_DFLAG_SvWR, + ATR_TYPE_STR, + PARENT_TYPE_JOB, + }, + + // JOB_ATR_mics_reserved + {(char *)ATTR_mics_reserved, /* "mics_reserved" */ + decode_str, + encode_str, + set_str, + comp_str, + free_str, + NULL_FUNC, + READ_ONLY | ATR_DFLAG_MOM | ATR_DFLAG_OPWR | ATR_DFLAG_SvWR, + ATR_TYPE_STR, + PARENT_TYPE_JOB, + }, /* JOB_ATR_UNKN - THIS MUST BE THE LAST ENTRY */ { (char *)"_other_", diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 38d08bb4d4..1f5fe5f0ab 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -1904,7 +1904,7 @@ int gpu_count( sprintf(log_buf, "Counted %d gpus %s on node %s that was skipped", count, - (freeonly? "free":"available"), + (freeonly? "free" : "total"), pnode->get_name()); log_ext(-1, __func__, log_buf, LOG_DEBUG); @@ -1922,7 +1922,7 @@ int gpu_count( sprintf(log_buf, "Counted %d gpus %s on node %s", count, - (freeonly? "free":"available"), + (freeonly? "free" : "total"), pnode->get_name()); log_ext(-1, __func__, log_buf, LOG_DEBUG); @@ -2402,7 +2402,8 @@ bool node_is_spec_acceptable( (gpu_req > gpu_free) || (mic_req > mic_free)) return(false); - if(job_is_exclusive) + + if (job_is_exclusive) { if(pnode->nd_slots.get_number_free() != pnode->nd_slots.get_total_execution_slots()) { @@ -2415,7 +2416,6 @@ bool node_is_spec_acceptable( - int parse_req_data( complete_spec_data &all_reqs) @@ -2992,7 +2992,7 @@ int select_nodes_using_hostlist( return(-2); } - if (node_is_spec_acceptable(pnode, req, ProcBMStr, eligible_nodes,job_is_exclusive) == false) + if (node_is_spec_acceptable(pnode, req, ProcBMStr, eligible_nodes, job_is_exclusive) == false) { snprintf(log_buf, sizeof(log_buf), "Requested node '%s' is not currently available", req.plist[0].name.c_str()); @@ -3963,57 +3963,62 @@ int place_mics_in_hostlist( #ifdef PENABLE_LINUX_CGROUPS + /* - * save_cpus_and_memory_cpusets() - * - * Adds the cpus and mems to the job's list */ -void save_cpus_and_memory_cpusets( +void save_cgroup_string_attr( job *pjob, const char *node_name, - std::string &cpus, - std::string &mems) + std::string &value, + int index) { - if (pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str == NULL) - { - std::string formatted(node_name); - if (cpus.size() != 0) - formatted += ":" + cpus; - pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str = strdup(formatted.c_str()); - pjob->ji_wattr[JOB_ATR_cpuset_string].at_flags |= ATR_VFLAG_SET; - } - else - { - std::string all_cpus = pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str; - all_cpus += "+"; - all_cpus += node_name; - if (cpus.size() != 0) - all_cpus += ":" + cpus; - free(pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str); - pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str = strdup(all_cpus.c_str()); - } - - if (pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str == NULL) + if (pjob->ji_wattr[index].at_val.at_str == NULL) { std::string formatted(node_name); - if (mems.size() != 0) - formatted += ":" + mems; - pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str = strdup(formatted.c_str()); - pjob->ji_wattr[JOB_ATR_memset_string].at_flags |= ATR_VFLAG_SET; + + if (value.size() != 0) + formatted += ":" + value; + + pjob->ji_wattr[index].at_val.at_str = strdup(formatted.c_str()); + pjob->ji_wattr[index].at_flags |= ATR_VFLAG_SET; } else { - std::string all_mems = pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str; - all_mems += "+"; - all_mems += node_name; - if (mems.size() != 0) - all_mems += ":" + mems; - free(pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str); - pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str = strdup(all_mems.c_str()); + std::string concat = pjob->ji_wattr[index].at_val.at_str; + concat += "+"; + concat += node_name; + if (value.size() != 0) + concat += ":" + value; + + free(pjob->ji_wattr[index].at_val.at_str); + pjob->ji_wattr[index].at_val.at_str = strdup(concat.c_str()); } + } // END save_cgroup_string_attr() + +/* + * save_cpus_and_memory_cpusets() + * + * Adds the cpus and mems to the job's list + */ + +void save_cpus_and_memory_cpusets( + + job *pjob, + const char *node_name, + cgroup_info &cgi) + + { + save_cgroup_string_attr(pjob, node_name, cgi.cpu_string, JOB_ATR_cpuset_string); + save_cgroup_string_attr(pjob, node_name, cgi.mem_string, JOB_ATR_memset_string); + + if (cgi.gpu_string.size() > 0) + save_cgroup_string_attr(pjob, node_name, cgi.gpu_string, JOB_ATR_gpus_reserved); + + if (cgi.mic_string.size() > 0) + save_cgroup_string_attr(pjob, node_name, cgi.mic_string, JOB_ATR_mics_reserved); } // END save_cpus_and_memory_cpusets() @@ -4177,7 +4182,8 @@ int place_subnodes_in_hostlist( char *ProcBMStr) { - int rc = PBSE_NONE; + int rc = PBSE_NONE; + char log_buf[LOCAL_LOG_BUF_SIZE]; #ifdef GEOMETRY_REQUESTS if (IS_VALID_STR(ProcBMStr)) { @@ -4217,8 +4223,7 @@ int place_subnodes_in_hostlist( pnode->nd_state |= INUSE_JOB; #ifdef PENABLE_LINUX_CGROUPS - std::string cpus; - std::string mems; + cgroup_info cgi; bool legacy_vmem = false; get_svr_attr_b(SRV_ATR_LegacyVmem, &legacy_vmem); @@ -4230,9 +4235,16 @@ int place_subnodes_in_hostlist( update_req_hostlist(pjob, pnode->get_name(), naji.req_index, naji.ppn_needed); - rc = pnode->nd_layout.place_job(pjob, cpus, mems, pnode->get_name(), legacy_vmem); + rc = pnode->nd_layout.place_job(pjob, cgi, pnode->get_name(), legacy_vmem); if (rc != PBSE_NONE) + { + snprintf(log_buf, sizeof(log_buf), + "Couldn't place job %s on node %s.", + pjob->ji_qs.ji_jobid, pnode->get_name()); + log_err(-1, __func__, log_buf); + return(rc); + } if ((pjob->ji_wattr[JOB_ATR_node_exclusive].at_flags & ATR_VFLAG_SET) && (pjob->ji_wattr[JOB_ATR_node_exclusive].at_val.at_long != 0) && @@ -4244,21 +4256,21 @@ int place_subnodes_in_hostlist( if (pnode->nd_layout.getTotalThreads() > 1) { sprintf(buf, "0-%d", pnode->nd_layout.getTotalThreads() - 1); - cpus = buf; + cgi.cpu_string = buf; } else - cpus = "0"; + cgi.cpu_string = "0"; if (pnode->nd_layout.getTotalChips() > 1) { sprintf(buf, "0-%d", pnode->nd_layout.getTotalChips() - 1); - mems = buf; + cgi.mem_string = buf; } else - mems = "0"; + cgi.mem_string = "0"; } - save_cpus_and_memory_cpusets(pjob, pnode->get_name(), cpus, mems); + save_cpus_and_memory_cpusets(pjob, pnode->get_name(), cgi); save_node_usage(pnode); #endif @@ -4267,6 +4279,11 @@ int place_subnodes_in_hostlist( } else { + snprintf(log_buf, sizeof(log_buf), + "Node %s doesn't have enough execution slots remaining for job %s.", + pnode->get_name(), pjob->ji_qs.ji_jobid); + log_err(-1, __func__, log_buf); + rc = -1; } diff --git a/src/test/job_func/test_uut.c b/src/test/job_func/test_uut.c index cb78ea1dde..41595192d1 100644 --- a/src/test/job_func/test_uut.c +++ b/src/test/job_func/test_uut.c @@ -366,8 +366,9 @@ START_TEST(svr_job_purge_test) test_job = job_alloc(); test_job->ji_qs.ji_substate = JOB_SUBSTATE_QUEUED; test_job->ji_qs.ji_state = JOB_STATE_QUEUED; + memset(test_job->ji_arraystructid, 0, sizeof(test_job->ji_arraystructid)); result = svr_job_purge(test_job); - fail_unless(result == 0, "non-queued job fail", result); + fail_unless(result == 0, "non-queued job fail: %d", result); // called_remove_job once means we didn't call job_free fail_unless(called_remove_job == 1); diff --git a/src/test/machine/scaffolding.c b/src/test/machine/scaffolding.c index 5774cbd41b..b470194669 100644 --- a/src/test/machine/scaffolding.c +++ b/src/test/machine/scaffolding.c @@ -353,7 +353,7 @@ void allocation::set_place_type(const std::string &place) this->place_type = my_placement_type; } -void allocation::place_indices_in_string(std::string &out, int which) {} +void allocation::place_indices_in_string(cgroup_info &cgi) {} int allocation::add_allocation(allocation const &a) { diff --git a/src/test/machine/test_uut.c b/src/test/machine/test_uut.c index ad7a1498ae..02de9daf32 100644 --- a/src/test/machine/test_uut.c +++ b/src/test/machine/test_uut.c @@ -70,8 +70,7 @@ START_TEST(test_place_all_execution_slots) { Machine m; job pjob; - std::string cpu; - std::string mem; + cgroup_info cgi; complete_req cr; pjob.ji_wattr[JOB_ATR_req_information].at_val.at_ptr = &cr; @@ -82,7 +81,7 @@ START_TEST(test_place_all_execution_slots) placed_all = 0; exec_slots = -1; num_for_host = 1; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(placed_all == 2, "placed all: %d", placed_all); } END_TEST @@ -92,8 +91,7 @@ START_TEST(test_spread_place) { Machine m; job pjob; - std::string cpu; - std::string mem; + cgroup_info cgi; complete_req cr; pjob.ji_wattr[JOB_ATR_req_information].at_val.at_ptr = &cr; @@ -108,12 +106,12 @@ START_TEST(test_spread_place) spreaded = true; // Make sure we call spread place once for each successfully placed task - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 1, "called %d", called_spread_place); num_for_host = 3; called_spread_place = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 3); // Now we're multiple instead of one so it should multiply the calls @@ -122,7 +120,7 @@ START_TEST(test_spread_place) numa_node_count = 2; num_for_host = 2; called_spread_place = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 4, "called %d times", called_spread_place); sockets = 2; @@ -130,7 +128,7 @@ START_TEST(test_spread_place) numa_node_count = 0; num_for_host = 3; called_spread_place = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 6); called_spread_place = 0; @@ -141,7 +139,7 @@ START_TEST(test_spread_place) // the req is set to need 10 memory and each socket is set to only have 5, so even // though we set the req to need one socket, we should get two - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 2, "called %d times", called_spread_place); req_mem = 0; @@ -309,8 +307,7 @@ END_TEST START_TEST(test_place_and_free_job) { - std::string cpu; - std::string mem; + cgroup_info cgi; Machine m; m.addSocket(2); job pjob; @@ -331,7 +328,7 @@ START_TEST(test_place_and_free_job) num_for_host = 4; num_tasks_fit = 4; num_placed = 4; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_place_task == 2, "Expected 2 calls but got %d", called_place_task); std::vector job_ids; @@ -350,7 +347,7 @@ START_TEST(test_place_and_free_job) num_tasks_fit = 4; num_placed = 4; called_place_task = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_place_task == 2, "Expected 2 calls but got %d", called_place_task); num_tasks_fit = 0; @@ -361,7 +358,7 @@ START_TEST(test_place_and_free_job) partially_placed = true; called_partially_place = 0; called_fits_on_socket = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_partially_place == 1, "called %d", called_partially_place); fail_unless(called_fits_on_socket == 1); @@ -371,7 +368,7 @@ START_TEST(test_place_and_free_job) my_req_count = 1; num_for_host = 1; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_partially_place == 2, "called %d", called_partially_place); fail_unless(called_fits_on_socket == 3, "called %d times", called_fits_on_socket); } diff --git a/src/test/node_manager/scaffolding.c b/src/test/node_manager/scaffolding.c index c69aded719..8d03fc30a2 100644 --- a/src/test/node_manager/scaffolding.c +++ b/src/test/node_manager/scaffolding.c @@ -981,11 +981,10 @@ void Machine::displayAsJson(stringstream &out, bool include_jobs) const {} int Machine::place_job( - job *pjob, - string &cpu_string, - string &mem_string, - const char *hostname, - bool legacy_vmem) + job *pjob, + cgroup_info &cgi, + const char *hostname, + bool legacy_vmem) { return(0); diff --git a/src/test/node_manager/test_uut.c b/src/test/node_manager/test_uut.c index 89f62aef1f..bb8933d994 100644 --- a/src/test/node_manager/test_uut.c +++ b/src/test/node_manager/test_uut.c @@ -143,24 +143,29 @@ START_TEST(check_node_jobs_exitence_test) END_TEST #ifdef PENABLE_LINUX_CGROUPS -void save_cpus_and_memory_cpusets(job *pjob, const char *host, std::string &cpus, std::string &mems); +void save_cpus_and_memory_cpusets(job *pjob, const char *host, cgroup_info &cgi); START_TEST(test_save_cpus_and_memory_cpusets) { job *pjob = new job(); - std::string cpus("0-3"); - std::string mems("0"); + cgroup_info cgi; + cgi.cpu_string = "0-3"; + cgi.mem_string = "0"; - save_cpus_and_memory_cpusets(pjob, "napali", cpus, mems); + save_cpus_and_memory_cpusets(pjob, "napali", cgi); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str, "napali:0-3")); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str, "napali:0")); - save_cpus_and_memory_cpusets(pjob, "wailua", cpus, mems); + save_cpus_and_memory_cpusets(pjob, "wailua", cgi); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str, "napali:0-3+wailua:0-3")); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str, "napali:0+wailua:0")); - - save_cpus_and_memory_cpusets(pjob, "waimea", cpus, mems); + + cgi.gpu_string = "0-1"; + cgi.mic_string = "4"; + save_cpus_and_memory_cpusets(pjob, "waimea", cgi); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str, "napali:0-3+wailua:0-3+waimea:0-3")); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str, "napali:0+wailua:0+waimea:0")); + fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_gpus_reserved].at_val.at_str, "waimea:0-1")); + fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_mics_reserved].at_val.at_str, "waimea:4")); } END_TEST From e9bbfbb8da2d0e290c51193d17d74960ae998c5b Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 20 Mar 2017 16:32:40 -0600 Subject: [PATCH 199/312] TRQ-3961. Fix restricting devices with the cgroup. --- src/resmom/trq_cgroups.c | 153 ++++++++++++--------------------------- 1 file changed, 47 insertions(+), 106 deletions(-) diff --git a/src/resmom/trq_cgroups.c b/src/resmom/trq_cgroups.c index d17cc3475f..465a4d3133 100644 --- a/src/resmom/trq_cgroups.c +++ b/src/resmom/trq_cgroups.c @@ -2034,62 +2034,6 @@ void trq_cg_delete_job_cgroups( } // END trq_cg_delete_job_cgroups() -/* - * trq_cg_set_forbidden_devices - * - * Add the indices in the forbidden list to the devices.deny file - * - * @param - forbidden_devices - gpus that are not to be used - * by this job - * @param - job_devices_path - path to devices directory for the job - * - */ - -int trq_cg_set_forbidden_devices(std::vector &forbidden_devices, std::string job_devices_path) - { - int rc; - char log_buf[LOCAL_LOG_BUF_SIZE]; - std::string devices_deny; - FILE *f; - - devices_deny = job_devices_path + '/' + "devices.deny"; - - rc = trq_cg_mkdir(job_devices_path.c_str(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); - if ((rc != 0) && - (errno != EEXIST)) - { - sprintf(log_buf, "failed to make directory %s for cgroup: %s", job_devices_path.c_str(), strerror(errno)); - log_err(errno, __func__, log_buf); - return(PBSE_SYSTEM); - } - - for (std::vector::iterator it = forbidden_devices.begin(); it != forbidden_devices.end(); it++) - { - char restricted_gpu[1024]; - -#ifdef NVIDIA_GPUS - sprintf(restricted_gpu, "echo \"c 195:%d rwm\" > %s", *it, devices_deny.c_str()); -#endif - -#ifdef MIC - sprintf(restricted_gpu, "echo \"c 245:%d rwm\" > %s", *it, devices_deny.c_str()); -#endif - - f = popen(restricted_gpu, "r"); - if (f == NULL) - { - sprintf(log_buf, "Failed to add restricted gpu to devices.deny list: index %d in %s", *it, job_devices_path.c_str()); - log_err(errno, __func__, log_buf); - return(PBSE_SYSTEM); - } - - pclose(f); - } - - return(PBSE_NONE); - } - - /* * trq_cg_add_devices_to_cgroups @@ -2110,7 +2054,6 @@ int trq_cg_add_devices_to_cgroup( std::vector device_indices; std::vector forbidden_devices; std::string job_devices_path; - char log_buf[LOCAL_LOG_BUF_SIZE]; int rc = PBSE_NONE; unsigned int device_count = 0; @@ -2134,20 +2077,7 @@ int trq_cg_add_devices_to_cgroup( find_range_in_cpuset_string(gpus_reserved, gpu_range); translate_range_string_to_vector(gpu_range.c_str(), device_indices); } - else - { - for (unsigned int i = 0; i < device_count; i++) - forbidden_devices.push_back(i); - - rc = trq_cg_set_forbidden_devices(forbidden_devices, job_devices_path); - if (rc != PBSE_NONE) - { - sprintf(log_buf, "Failed to write devices.deny list for job %s", pjob->ji_qs.ji_jobid); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - return(rc); - } - } -#elif defined(MIC) +#else // We aren't here unless either NVIDIA or MICS are defined if (pjob->ji_wattr[JOB_ATR_mics_reserved].at_val.at_str != NULL) { std::string mics_reserved(pjob->ji_wattr[JOB_ATR_mics_reserved].at_val.at_str); @@ -2155,51 +2085,62 @@ int trq_cg_add_devices_to_cgroup( find_range_in_cpuset_string(mics_reserved, mic_range); translate_range_string_to_vector(mic_range.c_str(), device_indices); } - else - { - for (unsigned int i = 0; i < device_count; i++) - forbidden_devices.push_back(i); - - rc = trq_cg_set_forbidden_devices(forbidden_devices, job_devices_path); - if (rc != PBSE_NONE) - { - sprintf(log_buf, "Failed to write devices.deny list for job %s", pjob->ji_qs.ji_jobid); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - return(rc); - } - } #endif - /* device_indices has the list of gpus for this job. - make a list of gpus that are not for this job - using that list */ - for (unsigned int i = 0; i < device_count; i++) + // device_indices has the list of either gpus or mics for this job + std::string disallow_all("a *:* rwm"); + const int NUM_ALLOW_STATEMENTS = 3; + // These 3 statments allow use of all character devices except the GPUs (or except the MICs for + // that configuration). + const char *allow_statements[NUM_ALLOW_STATEMENTS] = { - bool found; + "b *:* rwm", // allow all block devices +#ifdef NVIDIA_GPUS + "c 1:194 rwm", + "c 196:* rwm", +#else + "c 1:244 rwm", + "c 246:* rwm", +#endif + }; + + std::string allowed = job_devices_path + "/devices.allow"; + std::string denied = job_devices_path + "/devices.deny"; + std::string error; - found = false; - for (size_t j = 0; j < device_indices.size(); j++) + rc = write_to_file(denied.c_str(), disallow_all, error); + + if (rc == PBSE_NONE) + { + for (int i = 0; i < NUM_ALLOW_STATEMENTS; i++) { - if (device_indices[j] == i) + if ((rc = write_to_file(allowed.c_str(), allow_statements[i], error)) != PBSE_NONE) { - found = true; + log_err(errno, __func__, error.c_str()); break; } } + + if (rc == PBSE_NONE) + { + char allow_device_buf[MAXLINE]; - if (found == false) - forbidden_devices.push_back(i); - } - - if (forbidden_devices.size() == 0) - return(PBSE_NONE); - - rc = trq_cg_set_forbidden_devices(forbidden_devices, job_devices_path); - if (rc != PBSE_NONE) - { - sprintf(log_buf, " 2 Failed to write devices.deny list for job %s", pjob->ji_qs.ji_jobid); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - return(rc); + for (size_t i = 0; i < device_indices.size(); i++) + { + // Write the allow statement for this device +#ifdef NVIDIA_GPUS + sprintf(allow_device_buf, "c 195:%d rwm", device_indices[i]); +#else + sprintf(allow_device_buf, "c 245:%d rwm", device_indices[i]); +#endif + rc = write_to_file(allowed.c_str(), allow_device_buf, error); + if (rc != PBSE_NONE) + { + log_err(errno, __func__, error.c_str()); + break; + } + } + } } return(rc); From d4c90f345c452298722f45993b0a00509fa8f208 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 21 Mar 2017 16:26:23 -0600 Subject: [PATCH 200/312] TRQ-3961. Fix some errors in the previous check-in. --- src/resmom/mom_main.c | 3 + src/resmom/trq_cgroups.c | 121 ++++++++++++++++++++++++-------- src/test/mom_main/scaffolding.c | 5 ++ 3 files changed, 98 insertions(+), 31 deletions(-) diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index 6e15c2032b..d6cb27cbe5 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -304,6 +304,7 @@ extern int shut_nvidia_nvml(); extern int check_nvidia_setup(); #endif /* NVIDIA_GPUS */ +int read_all_devices(); int send_join_job_to_a_sister(job *pjob, int stream, eventent *ep, tlist_head phead, int node_id); void prepare_child_tasks_for_delete(); static void mom_lock(int fds, int op); @@ -4709,6 +4710,8 @@ int cg_initialize_hwloc_topology() hwloc_free_xmlbuffer(topology, xml_buf); #endif + read_all_devices(); + unsigned long flags = HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM; flags |= HWLOC_TOPOLOGY_FLAG_IO_DEVICES; diff --git a/src/resmom/trq_cgroups.c b/src/resmom/trq_cgroups.c index 465a4d3133..eb0c12fa0c 100644 --- a/src/resmom/trq_cgroups.c +++ b/src/resmom/trq_cgroups.c @@ -64,6 +64,7 @@ const int MAX_WRITE_RETRIES = 5; #ifdef PENABLE_LINUX_CGROUPS extern Machine this_node; +std::set character_device_ids; /* This array tracks if all of the hierarchies are mounted we need to run our control groups */ @@ -2089,20 +2090,7 @@ int trq_cg_add_devices_to_cgroup( // device_indices has the list of either gpus or mics for this job std::string disallow_all("a *:* rwm"); - const int NUM_ALLOW_STATEMENTS = 3; - // These 3 statments allow use of all character devices except the GPUs (or except the MICs for - // that configuration). - const char *allow_statements[NUM_ALLOW_STATEMENTS] = - { - "b *:* rwm", // allow all block devices -#ifdef NVIDIA_GPUS - "c 1:194 rwm", - "c 196:* rwm", -#else - "c 1:244 rwm", - "c 246:* rwm", -#endif - }; + std::string allow_all_block("b *:* rwm"); std::string allowed = job_devices_path + "/devices.allow"; std::string denied = job_devices_path + "/devices.deny"; @@ -2112,27 +2100,17 @@ int trq_cg_add_devices_to_cgroup( if (rc == PBSE_NONE) { - for (int i = 0; i < NUM_ALLOW_STATEMENTS; i++) - { - if ((rc = write_to_file(allowed.c_str(), allow_statements[i], error)) != PBSE_NONE) - { - log_err(errno, __func__, error.c_str()); - break; - } - } - - if (rc == PBSE_NONE) + if ((rc = write_to_file(allowed.c_str(), allow_all_block, error)) != PBSE_NONE) + log_err(errno, __func__, error.c_str()); + else { char allow_device_buf[MAXLINE]; - for (size_t i = 0; i < device_indices.size(); i++) + for (std::set::iterator it = character_device_ids.begin(); + it != character_device_ids.end(); + it++) { - // Write the allow statement for this device -#ifdef NVIDIA_GPUS - sprintf(allow_device_buf, "c 195:%d rwm", device_indices[i]); -#else - sprintf(allow_device_buf, "c 245:%d rwm", device_indices[i]); -#endif + sprintf(allow_device_buf, "c %d:* rwm", *it); rc = write_to_file(allowed.c_str(), allow_device_buf, error); if (rc != PBSE_NONE) { @@ -2140,10 +2118,91 @@ int trq_cg_add_devices_to_cgroup( break; } } + + if (rc == PBSE_NONE) + { + for (size_t i = 0; i < device_indices.size(); i++) + { + // Write the allow statement for this device +#ifdef NVIDIA_GPUS + sprintf(allow_device_buf, "c 195:%d rwm", device_indices[i]); +#else + sprintf(allow_device_buf, "c 245:%d rwm", device_indices[i]); +#endif + rc = write_to_file(allowed.c_str(), allow_device_buf, error); + if (rc != PBSE_NONE) + { + log_err(errno, __func__, error.c_str()); + break; + } + } + + if (rc == PBSE_NONE) + { + // Allow any other non-gpu / mic devices to be used. Things like nvidiactl must be available + // to all. + for (int i = device_count; i < 256; i++) + { +#ifdef NVIDIA_GPUS + sprintf(allow_device_buf, "c 195:%d rwm", i); +#else + sprintf(allow_device_buf, "c 245:%d rwm", i); +#endif + rc = write_to_file(allowed.c_str(), allow_device_buf, error); + if (rc != PBSE_NONE) + { + log_err(errno, __func__, error.c_str()); + break; + } + } + } + } } } return(rc); } // END trq_cg_add_devices_to_cgroup() + + +int read_all_devices() + + { + int forbidden_device = 0; +#ifdef NVIDIA_GPUS + forbidden_device = 195; +#elif defined(MICS) + forbidden_device = 245; +#endif + + if (forbidden_device == 0) + return(PBSE_NONE); + +#ifdef NVIDIA_GPUS + // force the loading of nvidia-uvm + system("nvidia-modprobe -u -c=0"); +#endif + + std::ifstream infile("/proc/devices"); + std::string line; + + while (std::getline(infile, line)) + { + if (line == "Character devices:") + continue; + else if (line == "Block devices:") + break; + + int device_index; + std::istringstream iss(line); + + if ((iss >> device_index) && + (device_index != forbidden_device)) + character_device_ids.insert(device_index); + } + + return(PBSE_NONE); + } // END read_all_devices() + + #endif diff --git a/src/test/mom_main/scaffolding.c b/src/test/mom_main/scaffolding.c index 46b6c37542..43ed5b5e07 100644 --- a/src/test/mom_main/scaffolding.c +++ b/src/test/mom_main/scaffolding.c @@ -1136,3 +1136,8 @@ void authorized_hosts::add_authorized_address(unsigned long addr, unsigned short authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; + +int read_all_devices() + { + return(PBSE_NONE); + } From 7cf4aca8c6f8d887327b4cdc3d34891c148cd6cb Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 22 Mar 2017 11:56:55 -0600 Subject: [PATCH 201/312] TRQ-3965. Make pbs_server automatically take a new layout when the number of gpus has increased. --- src/server/process_mom_update.c | 3 ++- src/test/process_mom_update/scaffolding.c | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/server/process_mom_update.c b/src/server/process_mom_update.c index 304fdc79dc..962df66c54 100644 --- a/src/server/process_mom_update.c +++ b/src/server/process_mom_update.c @@ -774,7 +774,8 @@ void update_layout_if_needed( pnode->nd_layout = m; } - else if ((pnode->nd_layout.getTotalThreads() != pnode->nd_slots.get_total_execution_slots()) && + else if (((pnode->nd_layout.getTotalThreads() != pnode->nd_slots.get_total_execution_slots()) || + (pnode->nd_ngpus != pnode->nd_layout.get_total_gpus())) && (pnode->nd_job_usages.size() == 0)) { int old_count = pnode->nd_layout.getTotalThreads(); diff --git a/src/test/process_mom_update/scaffolding.c b/src/test/process_mom_update/scaffolding.c index aa360460f6..37a1a48872 100644 --- a/src/test/process_mom_update/scaffolding.c +++ b/src/test/process_mom_update/scaffolding.c @@ -688,5 +688,10 @@ Machine::Machine( { } +int Machine::get_total_gpus() const + { + return(0); + } + #endif From c5a509fc49ed129ddd78758fd96ff23077723bbd Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 22 Mar 2017 12:02:43 -0600 Subject: [PATCH 202/312] TRQ-3965. Add a unit test. --- src/test/process_mom_update/test_uut.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/process_mom_update/test_uut.c b/src/test/process_mom_update/test_uut.c index 8eed741992..ded0e03d1b 100644 --- a/src/test/process_mom_update/test_uut.c +++ b/src/test/process_mom_update/test_uut.c @@ -56,6 +56,11 @@ START_TEST(test_update_layout_if_needed) update_layout_if_needed(&pnode, with_threads, false); update_layout_if_needed(&pnode, with_threads, false); fail_unless(event_logged == 1, "event logged: %d", event_logged); + + // Make sure that changing the number of gpus on the node triggers an event + pnode.nd_ngpus = 1; + update_layout_if_needed(&pnode, with_threads, false); + fail_unless(event_logged == 2, "event logged: %d", event_logged); } END_TEST #endif From 97a14a9d31a6c76914f7e5cb532f535362fca414 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 22 Mar 2017 16:11:53 -0600 Subject: [PATCH 203/312] TRQ-3961. Fix a regression around MICS. --- src/resmom/trq_cgroups.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resmom/trq_cgroups.c b/src/resmom/trq_cgroups.c index eb0c12fa0c..a7a6eae445 100644 --- a/src/resmom/trq_cgroups.c +++ b/src/resmom/trq_cgroups.c @@ -2171,7 +2171,7 @@ int read_all_devices() int forbidden_device = 0; #ifdef NVIDIA_GPUS forbidden_device = 195; -#elif defined(MICS) +#elif defined(MIC) forbidden_device = 245; #endif From 9f659502b0d08de898d51fe6cffe35927714c27f Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 24 Mar 2017 10:35:52 -0600 Subject: [PATCH 204/312] TRQ-3807 - create sendmail_path sever attribute --- src/include/pbs_ifl.h | 1 + src/include/qmgr_svr_public.h | 1 + src/include/server.h | 1 + src/server/svr_attr_def.c | 13 +++++++++++++ src/server/svr_mail.c | 13 ++++++++++++- src/test/svr_mail/scaffolding.c | 7 +++++++ src/test/svr_mail/test_uut.c | 28 ++++++++++++++++++++++++++++ 7 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/include/pbs_ifl.h b/src/include/pbs_ifl.h index 2c772b792e..a345b7266d 100644 --- a/src/include/pbs_ifl.h +++ b/src/include/pbs_ifl.h @@ -341,6 +341,7 @@ #define ATTR_pass_cpu_clock "pass_cpu_clock" #define ATTR_request_version "request_version" #define ATTR_req_information "req_information" +#define ATTR_sendmail_path "sendmail_path" /* additional node "attributes" names */ #define ATTR_NODE_state "state" diff --git a/src/include/qmgr_svr_public.h b/src/include/qmgr_svr_public.h index ba2b95732b..c767944cbb 100644 --- a/src/include/qmgr_svr_public.h +++ b/src/include/qmgr_svr_public.h @@ -204,3 +204,4 @@ ATTR_ghost_array_recovery, ATTR_cgroup_per_task, ATTR_idle_slot_limit, ATTR_default_gpu_mode, +ATTR_sendmail_path, diff --git a/src/include/server.h b/src/include/server.h index 9f3215bab1..43171243f1 100644 --- a/src/include/server.h +++ b/src/include/server.h @@ -229,6 +229,7 @@ enum srv_atr SRV_ATR_CgroupPerTask, SRV_ATR_IdleSlotLimit, SRV_ATR_DefaultGpuMode, + SRV_ATR_SendmailPath, /* This must be last */ SRV_ATR_LAST diff --git a/src/server/svr_attr_def.c b/src/server/svr_attr_def.c index e42cf65cf6..afba56807d 100644 --- a/src/server/svr_attr_def.c +++ b/src/server/svr_attr_def.c @@ -1651,4 +1651,17 @@ attribute_def svr_attr_def[] = PARENT_TYPE_SERVER }, + /* SRV_ATR_SendmailPath */ + {(char *)ATTR_sendmail_path, // "sendmail_path" + decode_str, + encode_str, + set_str, + comp_str, + free_str, + NULL_FUNC, + MGR_ONLY_SET, + ATR_TYPE_STR, + PARENT_TYPE_SERVER + }, + }; diff --git a/src/server/svr_mail.c b/src/server/svr_mail.c index d7ca7f2500..a057f2fa87 100644 --- a/src/server/svr_mail.c +++ b/src/server/svr_mail.c @@ -226,6 +226,7 @@ int get_sendmail_args( int rc = PBSE_NONE; int numargs = 0; const char *mailfrom = NULL; + const char *sendmail_path = NULL; char *mailptr; /* Who is mail from, if SRV_ATR_mailfrom not set use default */ @@ -247,7 +248,17 @@ int get_sendmail_args( } } - sendmail_args[numargs++] = (char *)SENDMAIL_CMD; + // see if sendmail path server attribute set + get_svr_attr_str(SRV_ATR_SendmailPath, (char **)&sendmail_path); + if (sendmail_path != NULL) + { + sendmail_args[numargs++] = (char *)sendmail_path; + } + else + { + sendmail_args[numargs++] = (char *)SENDMAIL_CMD; + } + sendmail_args[numargs++] = (char *)"-f"; sendmail_args[numargs++] = (char *)mailfrom; diff --git a/src/test/svr_mail/scaffolding.c b/src/test/svr_mail/scaffolding.c index b6556fd5cd..0a4eb390fb 100644 --- a/src/test/svr_mail/scaffolding.c +++ b/src/test/svr_mail/scaffolding.c @@ -26,6 +26,7 @@ pthread_mutex_t *listener_command_mutex; int listening_socket; threadpool_t *task_pool; int called; +bool get_svr_attr_str_return_svr_sm_attr = false; int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ @@ -146,6 +147,12 @@ int get_svr_attr_str( { static char *bodyfmt = strdup("Zaphod"); + if (get_svr_attr_str_return_svr_sm_attr) + { + *str = server.sv_attr[SRV_ATR_SendmailPath].at_val.at_str; + return(0); + } + if ((index == SRV_ATR_MailBodyFmt) && (empty_body == true)) { diff --git a/src/test/svr_mail/test_uut.c b/src/test/svr_mail/test_uut.c index 7997c4dba3..8b10d6aae5 100644 --- a/src/test/svr_mail/test_uut.c +++ b/src/test/svr_mail/test_uut.c @@ -10,6 +10,7 @@ #include "test_svr_mail.h" #include "mail_throttler.hpp" #include "work_task.h" +#include "server.h" @@ -27,8 +28,10 @@ extern int called; void send_email_batch(struct work_task *pwt); +int get_sendmail_args(char **, mail_info *, char **); extern mail_throttler pending_emails; extern bool empty_body; +extern bool get_svr_attr_str_return_svr_sm_attr; void init_server() { @@ -348,6 +351,27 @@ START_TEST(mail_point_p) } END_TEST +START_TEST(test_get_sendmail_args) + { + char *sendmail_args[100]; + mail_info mi; + char *p; + + get_sendmail_args(sendmail_args, &mi, (char **)&p); + + // expect value compiled in + fail_unless(strcmp(sendmail_args[0], UT_SENDMAIL_CMD) == 0); + + // now set the server attribute + server.sv_attr[SRV_ATR_SendmailPath].at_val.at_str = (char *)"./foo"; + + // expect to match server attribute (not one compiled in) + get_svr_attr_str_return_svr_sm_attr = true; + get_sendmail_args(sendmail_args, &mi, (char **)&p); + fail_unless(strcmp(sendmail_args[0], "./foo") == 0); + } +END_TEST + Suite *svr_mail_suite(void) { Suite *s = suite_create("svr_mail_suite methods"); @@ -368,6 +392,10 @@ Suite *svr_mail_suite(void) tcase_add_test(tc_core, mail_point_p); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_get_sendmail_args"); + tcase_add_test(tc_core, test_get_sendmail_args); + suite_add_tcase(s, tc_core); + return s; } From 369adafaa79f6e37c75c0ccfd40fb1a47039b2dd Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 24 Mar 2017 12:10:59 -0600 Subject: [PATCH 205/312] TRQ-3964 - enable core files by default in systemd unit files --- contrib/systemd/pbs_mom.service.in | 1 + contrib/systemd/pbs_server.service.in | 2 ++ contrib/systemd/trqauthd.service.in | 2 ++ 3 files changed, 5 insertions(+) diff --git a/contrib/systemd/pbs_mom.service.in b/contrib/systemd/pbs_mom.service.in index 09be264aab..a13513fb8d 100644 --- a/contrib/systemd/pbs_mom.service.in +++ b/contrib/systemd/pbs_mom.service.in @@ -11,6 +11,7 @@ User=root LimitNOFILE=32768 LimitMEMLOCK=infinity LimitSTACK=12582912 +LimitCORE=infinity Environment=PBS_HOME=@PBS_HOME@ Environment=PBS_ARGS= diff --git a/contrib/systemd/pbs_server.service.in b/contrib/systemd/pbs_server.service.in index 30e5197459..0cdb79a911 100644 --- a/contrib/systemd/pbs_server.service.in +++ b/contrib/systemd/pbs_server.service.in @@ -8,6 +8,8 @@ After=trqauthd.service network.target local-fs.target rsyslog.target Type=simple User=root +LimitCORE=infinity + # Let systemd guess the pid. # # PIDFile should not be set to PBS_HOME/server_priv/server.lock diff --git a/contrib/systemd/trqauthd.service.in b/contrib/systemd/trqauthd.service.in index 7c29a68e7f..0930010d60 100644 --- a/contrib/systemd/trqauthd.service.in +++ b/contrib/systemd/trqauthd.service.in @@ -8,6 +8,8 @@ After=network.target local-fs.target syslog.target Type=simple User=root +LimitCORE=infinity + ExecStart=@sbindir@/trqauthd -F ExecStop=-@sbindir@/trqauthd -d From 1b6990e88c82973b6c264839fc7618c455ced7a4 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 28 Mar 2017 11:54:39 -0600 Subject: [PATCH 206/312] TRQ-3934. Treat the memory value in req as per req and not per task. This caused some regressions in placing memory for jobs. Conflicts: src/include/machine.hpp --- src/include/machine.hpp | 3 +- src/lib/Libattr/req.cpp | 1 + src/lib/Libutils/allocation.cpp | 2 +- src/lib/Libutils/machine.cpp | 72 ++++++++++++++++++++++++++++++- src/lib/Libutils/numa_chip.cpp | 6 +-- src/test/allocation/scaffolding.c | 7 +++ src/test/allocation/test_uut.c | 10 +++++ src/test/machine/scaffolding.c | 10 +++++ src/test/numa_chip/scaffolding.c | 9 +++- src/test/numa_chip/test_uut.c | 11 +++++ 10 files changed, 123 insertions(+), 8 deletions(-) diff --git a/src/include/machine.hpp b/src/include/machine.hpp index 289e9bac3a..3523c813de 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -337,7 +337,7 @@ class Machine int get_total_gpus() const; int getAvailableSockets() const; int getAvailableChips() const; - int getAvailableMemory() const; + hwloc_uint64_t getAvailableMemory() const; int getAvailableCores() const; int getAvailableThreads() const; int getDedicatedSockets() const; @@ -367,6 +367,7 @@ class Machine bool check_if_possible(int &sockets, int &numa_nodes, int &cores, int &threads) const; bool is_initialized() const; void reinitialize_from_json(const std::string &json_layout, std::vector &valid_ids); + void compare_remaining_values(allocation &remaining, const char *caller) const; }; extern Machine this_node; diff --git a/src/lib/Libattr/req.cpp b/src/lib/Libattr/req.cpp index 6380d3c4bf..ec98e8089b 100644 --- a/src/lib/Libattr/req.cpp +++ b/src/lib/Libattr/req.cpp @@ -2550,6 +2550,7 @@ void req::clear_allocations() { this->task_allocations.clear(); + this->hostlist.clear(); } // END clear_allocations() diff --git a/src/lib/Libutils/allocation.cpp b/src/lib/Libutils/allocation.cpp index 15e8bd591c..0a2a6e3b92 100644 --- a/src/lib/Libutils/allocation.cpp +++ b/src/lib/Libutils/allocation.cpp @@ -159,7 +159,7 @@ allocation::allocation( { this->cpus = r.getExecutionSlots(); - this->memory = r.getMemory(); + this->memory = r.getMemory() / r.getTaskCount(); this->gpus = r.get_gpus(); this->mics = r.getMics(); diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index ec1df4d32a..6e74f1491b 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -427,6 +427,16 @@ hwloc_uint64_t Machine::getTotalMemory() const return(this->totalMemory); } +hwloc_uint64_t Machine::getAvailableMemory() const + { + hwloc_uint64_t available = 0; + + for (size_t i = 0; i < this->sockets.size(); i++) + available += this->sockets[i].getAvailableMemory(); + + return(available); + } + int Machine::getTotalSockets() const { return(this->totalSockets); @@ -621,6 +631,61 @@ void Machine::place_all_execution_slots( +/* + * compare_remaining_values() + * + * Looks at what couldn't be placed in the allocation remaining and logs an appropriate message. + * @param remaining - the allocation that couldn't be completely placed. + * @param caller - the name of the calling function. + */ + +void Machine::compare_remaining_values( + + allocation &remaining, + const char *caller) const + + { + char log_buf[LOCAL_LOG_BUF_SIZE]; + std::string error; + + if (remaining.cpus != 0) + { + sprintf(log_buf, "Couldn't place %d cpus, have %d/%d cores and threads available. ", + remaining.cpus, this->getAvailableCores(), this->getAvailableThreads()); + error += log_buf; + } + + if (remaining.memory != 0) + { + sprintf(log_buf, "Couldn't place %lukb memory, have %lukb available. ", + remaining.memory, (unsigned long)this->getAvailableMemory()); + error += log_buf; + } + + if (remaining.mics != 0) + { + sprintf(log_buf, "Couldn't place %d mics. ", remaining.mics); + error += log_buf; + } + + if (remaining.gpus != 0) + { + sprintf(log_buf, "Couldn't place %d gpus. ", remaining.gpus); + error += log_buf; + } + + if (remaining.place_cpus != 0) + { + sprintf(log_buf, "Couldn't place %d place cpus, have %d/%d cores and threads available. ", + remaining.place_cpus, this->getAvailableCores(), this->getAvailableThreads()); + error += log_buf; + } + + log_err(-1, caller, error.c_str()); + } // END compare_remaining_values() + + + /* * place_remaining() * @@ -737,9 +802,12 @@ void Machine::place_remaining( master.set_place_type(place_legacy); } + // If it didn't get placed, log what prevented it from being placed if (fit_somewhere == false) + { + compare_remaining_values(remaining, __func__); break; - + } remaining_tasks--; } @@ -881,7 +949,7 @@ int Machine::spread_place( else { // Make sure we are grabbing enough memory - unsigned long mem_needed = r.getMemory(); + unsigned long mem_needed = r.getMemory() / r.getTaskCount(); unsigned long mem_count = 0; int mem_quantity = 0; diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index a76ae92954..50e4374ec4 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -964,7 +964,7 @@ double Chip::how_many_tasks_fit( else cpu_tasks = this->availableThreads / max_cpus; - unsigned long memory = r.getMemory(); + unsigned long memory = r.getMemory() / r.getTaskCount(); // Memory isn't required for submission if (memory != 0) @@ -1369,7 +1369,7 @@ bool Chip::task_will_fit( { bool fits = false; int max_cpus = r.getExecutionSlots(); - hwloc_uint64_t mem_per_task = r.getMemory(); + hwloc_uint64_t mem_per_task = r.getMemory() / r.getTaskCount(); int gpus_per_task = r.get_gpus(); int mics_per_task = r.getMics(); bool cores_only = (r.getThreadUsageString() == use_cores); @@ -1995,7 +1995,7 @@ int Chip::place_task( allocation chip_alloc(master.jobid.c_str()); int tasks_placed = 0; int execution_slots_per_task = r.getExecutionSlots(); - hwloc_uint64_t mem_per_task = r.getMemory(); + hwloc_uint64_t mem_per_task = r.getMemory() / r.getTaskCount(); int practical_place = master.place_type; chip_alloc.place_type = master.place_type; diff --git a/src/test/allocation/scaffolding.c b/src/test/allocation/scaffolding.c index acca1c2788..0be97352a1 100644 --- a/src/test/allocation/scaffolding.c +++ b/src/test/allocation/scaffolding.c @@ -14,6 +14,8 @@ const char *place_legacy2 = "legacy2"; const char *use_cores = "usecores"; +int tc = 1; + std::string req::getPlacementType() const { return(""); @@ -33,6 +35,11 @@ int req::getExecutionSlots() const return(2); } +int req::getTaskCount() const + { + return(tc); + } + std::string req::getThreadUsageString() const { diff --git a/src/test/allocation/test_uut.c b/src/test/allocation/test_uut.c index a645a846e3..822bb2b991 100644 --- a/src/test/allocation/test_uut.c +++ b/src/test/allocation/test_uut.c @@ -6,6 +6,8 @@ #include "allocation.hpp" #include "req.hpp" +extern int tc; + START_TEST(test_write_task_information) { @@ -67,11 +69,19 @@ START_TEST(test_allocation_constructors) allocation a3("1.napali"); fail_unless(a3.jobid == "1.napali"); + tc = 1; req r; allocation a4(r); fail_unless(a4.memory == 1024, "mem = %d", a.memory); fail_unless(a4.cpus == 2); fail_unless(a4.cores_only == true); + + // Make sure that we show memory per task + tc = 2; + allocation a5(r); + fail_unless(a5.memory == 512, "mem = %d", a.memory); + fail_unless(a5.cpus == 2); + fail_unless(a5.cores_only == true); } END_TEST diff --git a/src/test/machine/scaffolding.c b/src/test/machine/scaffolding.c index b470194669..bb50329613 100644 --- a/src/test/machine/scaffolding.c +++ b/src/test/machine/scaffolding.c @@ -68,6 +68,11 @@ hwloc_uint64_t Socket::getMemory() const return(sock_mem); } +hwloc_uint64_t Socket::getAvailableMemory() const + { + return(sock_mem); + } + hwloc_uint64_t Socket::get_memory_for_completely_free_chips( unsigned long diff, @@ -254,6 +259,11 @@ unsigned long req::getMemory() const return(req_mem); } +int req::getTaskCount() const + { + return(1); + } + int req::getPlaceCores() const { return(my_core_count); diff --git a/src/test/numa_chip/scaffolding.c b/src/test/numa_chip/scaffolding.c index 51089f7fb7..d460aad0c6 100644 --- a/src/test/numa_chip/scaffolding.c +++ b/src/test/numa_chip/scaffolding.c @@ -137,13 +137,18 @@ int get_machine_total_memory(hwloc_topology_t topology, unsigned long *memory) return(PBSE_NONE); } -req::req() : mem(0), cores(0), threads(0), gpus(0), mics(0) {} +req::req() : mem(0), cores(0), threads(0), gpus(0), mics(0), task_count(1) {} unsigned long req::getMemory() const { return(this->mem); } +int req::getTaskCount() const + { + return(this->task_count); + } + int req::getPlaceCores() const { return(this->cores); @@ -180,6 +185,8 @@ int req::set_value(const char *name, const char *value, bool is_default) this->gpus = atoi(value); else if (!strcmp(name, MICS)) this->mics = atoi(value); + else if (!strcmp(name, "task_count")) + this->task_count = atoi(value); return(0); } diff --git a/src/test/numa_chip/test_uut.c b/src/test/numa_chip/test_uut.c index 9ea16b491c..65a765750c 100644 --- a/src/test/numa_chip/test_uut.c +++ b/src/test/numa_chip/test_uut.c @@ -741,6 +741,17 @@ START_TEST(test_how_many_tasks_fit) c.setMemory(5); tasks = c.how_many_tasks_fit(r, 0); fail_unless(tasks == 5, "%d tasks fit, expected 5", tasks); + + // Make sure we handle memory per task + req rpt; + rpt.set_value("lprocs", "1", false); + rpt.set_value("memory", "5kb", false); + tasks = c.how_many_tasks_fit(rpt, 0); + fail_unless(tasks == 1, "%d tasks fit, expected 1", tasks); + + rpt.set_value("task_count", "5", false); + tasks = c.how_many_tasks_fit(rpt, 0); + fail_unless(tasks == 5, "%d tasks fit, expected 5", tasks); thread_type = use_cores; // Cores are currently 0 From 7ec3ed13175a050ee3ef149dd52c49741bc7ae64 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 28 Mar 2017 14:32:34 -0600 Subject: [PATCH 207/312] TRQ-3852 - fix path problem in RPM spec files --- buildutils/torque.adaptive.spec.el6 | 6 +++--- buildutils/torque.adaptive.spec.el7 | 6 +++--- buildutils/torque.adaptive.spec.sles12 | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/buildutils/torque.adaptive.spec.el6 b/buildutils/torque.adaptive.spec.el6 index cd216f4bb2..32dd2f847a 100644 --- a/buildutils/torque.adaptive.spec.el6 +++ b/buildutils/torque.adaptive.spec.el6 @@ -603,10 +603,10 @@ echo '$logevent 225 # bitmap of which events to log' > \ %{buildroot}%{torque_sysconfdir}/mom_priv/config %{__mkdir_p} %{buildroot}/etc/profile.d -echo 'test -d ${_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh +echo 'test -d %{_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh echo 'test `id -u` = 0 && test -d %{_sbindir} && export PATH=%{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.sh -echo 'test -d ${_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh -echo 'test `id -u` = 0 && test-d ${_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh +echo 'test -d %{_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh +echo 'test `id -u` = 0 && test -d %{_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh # Moab requires libtorque.so.0, but works with libtorque.so.2, so fudge it %{__ln_s} libtorque.so.2 %{buildroot}%{_libdir}/libtorque.so.0 diff --git a/buildutils/torque.adaptive.spec.el7 b/buildutils/torque.adaptive.spec.el7 index e8793628d8..998aab355b 100644 --- a/buildutils/torque.adaptive.spec.el7 +++ b/buildutils/torque.adaptive.spec.el7 @@ -602,10 +602,10 @@ echo '$logevent 225 # bitmap of which events to log' > \ %{buildroot}%{torque_sysconfdir}/mom_priv/config %{__mkdir_p} %{buildroot}/etc/profile.d -echo 'test -d ${_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh +echo 'test -d %{_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh echo 'test `id -u` = 0 && test -d %{_sbindir} && export PATH=%{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.sh -echo 'test -d ${_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh -echo 'test `id -u` = 0 && test-d ${_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh +echo 'test -d %{_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh +echo 'test `id -u` = 0 && test -d %{_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh # Moab requires libtorque.so.0, but works with libtorque.so.2, so fudge it %{__ln_s} libtorque.so.2 %{buildroot}%{_libdir}/libtorque.so.0 diff --git a/buildutils/torque.adaptive.spec.sles12 b/buildutils/torque.adaptive.spec.sles12 index 18ebaeecd6..9eeaa7d802 100644 --- a/buildutils/torque.adaptive.spec.sles12 +++ b/buildutils/torque.adaptive.spec.sles12 @@ -602,10 +602,10 @@ echo '$logevent 225 # bitmap of which events to log' > \ %{buildroot}%{torque_sysconfdir}/mom_priv/config %{__mkdir_p} %{buildroot}/etc/profile.d -echo 'test -d ${_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh +echo 'test -d %{_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh echo 'test `id -u` = 0 && test -d %{_sbindir} && export PATH=%{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.sh -echo 'test -d ${_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh -echo 'test `id -u` = 0 && test-d ${_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh +echo 'test -d %{_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh +echo 'test `id -u` = 0 && test -d %{_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh # Moab requires libtorque.so.0, but works with libtorque.so.2, so fudge it %{__ln_s} libtorque.so.2 %{buildroot}%{_libdir}/libtorque.so.0 From f2638b4cda8090a784167822a2f74c9c078d67c7 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 28 Mar 2017 16:53:15 -0600 Subject: [PATCH 208/312] TRQ-2978. Fix one place check_pwd()'s return was being evaluated incorrectly. --- src/resmom/mom_comm.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index 7883fefca0..64da90e152 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -2629,10 +2629,7 @@ int im_join_job_as_sister( } } - bool good; - - good = check_pwd(pjob); - if (good == false) + if (check_pwd(pjob) != PBSE_NONE) { /* log_buffer populated in check_pwd() */ From 28238eff7cb974cdda0fc38342229e1700fcf92a Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 30 Mar 2017 14:38:14 -0600 Subject: [PATCH 209/312] TRQ-3979 and github issue #421. Fix a place where an unlocked job is unlocked again. This causes a crash in recent implementations of the pthread library. --- src/server/req_register.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/req_register.c b/src/server/req_register.c index 974fded556..6944c9d817 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -3401,7 +3401,9 @@ void removeBeforeAnyDependencies( job *pLockedJob = *pjob_ptr; std::string jobid(pLockedJob->ji_qs.ji_jobid); - mutex_mgr job_mutex(pLockedJob->ji_mutex,true); + mutex_mgr job_mutex(pLockedJob->ji_mutex, true); + job_mutex.set_unlock_on_exit(false); + pbs_attribute *pattr = &pLockedJob->ji_wattr[JOB_ATR_depend]; struct depend *pDep = find_depend(JOB_DEPEND_TYPE_BEFOREANY,pattr); From 7537312f1c1071d9ae295c8a6ce61edf5d6c5557 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 30 Mar 2017 16:36:27 -0600 Subject: [PATCH 210/312] TRQ-3825. Verify the correct value. Patch courtesy of Isaac daMan Tobler. --- src/server/resc_def_all.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/server/resc_def_all.c b/src/server/resc_def_all.c index 47fb8a16a5..658a327eb1 100644 --- a/src/server/resc_def_all.c +++ b/src/server/resc_def_all.c @@ -1471,10 +1471,12 @@ int verify_as_time( int mode) { - if (pattr->at_val.at_str != NULL) + if (r->rs_value.at_val.at_str != NULL) { - if (time_str_to_seconds(pattr->at_val.at_str) < 0) + if (time_str_to_seconds(r->rs_value.at_val.at_str) < 0) + { return(-1); + } } return(PBSE_NONE); From 3383bdfc7f56ae57aaba040b085c5e99dedbba72 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 3 Apr 2017 13:47:15 -0600 Subject: [PATCH 211/312] TRQ-3932. To avoid cheating max_user_queuable, only route whole arrays. --- src/include/pbs_job.h | 1 + src/server/job.cpp | 2 +- src/server/job_func.c | 53 +++++++++++++++----------- src/server/job_route.c | 81 +++++++++++++++++++++++++++++++--------- src/server/pbsd_main.c | 36 ------------------ src/server/svr_jobfunc.c | 9 +++-- 6 files changed, 103 insertions(+), 79 deletions(-) diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 98a33aa1e9..cacd66ec9e 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -809,6 +809,7 @@ class job unsigned ji_queue_counted; bool ji_being_deleted; int ji_commit_done; /* req_commit has completed. If in routing queue job can now be routed */ + bool ji_routed; // false if the job is an array template that is stuck in a routing queue. /* * fixed size internal data - maintained via "quick save" diff --git a/src/server/job.cpp b/src/server/job.cpp index 3024161083..5506352d6b 100644 --- a/src/server/job.cpp +++ b/src/server/job.cpp @@ -14,7 +14,7 @@ job::job() : ji_plugin_usage_info(), ji_momstat(0), ji_modified(0), ji_momhandle ji_have_nodes_request(false), ji_external_clone(NULL), ji_cray_clone(NULL), ji_parent_job(NULL), ji_internal_id(-1), ji_being_recycled(false), ji_last_reported_time(0), ji_mod_time(0), - ji_queue_counted(0), ji_being_deleted(false), ji_commit_done(false) + ji_queue_counted(0), ji_being_deleted(false), ji_commit_done(false), ji_routed(true) { memset(this->ji_arraystructid, 0, sizeof(ji_arraystructid)); diff --git a/src/server/job_func.c b/src/server/job_func.c index 8240e6131e..08fe20f5bd 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -155,6 +155,7 @@ #define FALSE 0 #endif +#define ARRAY_ROUTING_RETRY_SECONDS 45 #define MAXLINE 1024 extern int LOGLEVEL; @@ -196,8 +197,6 @@ extern char *path_jobinfo_log; extern char *log_file; extern char *job_log_file; -extern sem_t *job_clone_semaphore; - static void send_qsub_delmsg( job *pjob, /* I */ @@ -1194,7 +1193,6 @@ int create_and_queue_array_subjob( if ((pa = get_array(arrayid.c_str())) == NULL) { - sem_wait(job_clone_semaphore); return(FATAL_ERROR); } @@ -1211,7 +1209,6 @@ int create_and_queue_array_subjob( clone_mgr.set_unlock_on_exit(false); } - sem_wait(job_clone_semaphore); return(FATAL_ERROR); } @@ -1225,7 +1222,6 @@ int create_and_queue_array_subjob( if ((pa = get_array(arrayid.c_str())) == NULL) { - sem_wait(job_clone_semaphore); return(FATAL_ERROR); } @@ -1352,9 +1348,21 @@ int perform_array_postprocessing( -#ifndef CLONE_BATCH_SIZE -#define CLONE_BATCH_SIZE 256 -#endif /* CLONE_BATCH_SIZE */ +void job_clone_task_wrapper( + + work_task *ptask) + + { + char *job_id = (char *)ptask->wt_parm1; + + if (job_id != NULL) + { + job_clone_wt(job_id); + } + + free(ptask->wt_mutex); + free(ptask); + } // END job_clone_task_wrapper() @@ -1370,10 +1378,12 @@ void *job_clone_wt( job *template_job; char *jobid; int rc; - std::string prev_job_id; char namebuf[MAXPATHLEN]; job_array *pa; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + + std::string prev_job_id; std::string adjusted_path_jobs; jobid = (char *)cloned_id; @@ -1381,15 +1391,7 @@ void *job_clone_wt( if (jobid == NULL) { log_err(ENOMEM, __func__, "Can't malloc"); - return(NULL); - } - - /* increment the job_clone_semaphore so people - know we are making jobs for this array */ - rc = sem_post(job_clone_semaphore); - if (rc) - { - log_err(-1, __func__, "failed to post job_clone_semaphore"); + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); return(NULL); } @@ -1401,13 +1403,22 @@ void *job_clone_wt( if (template_job != NULL) unlock_ji_mutex(template_job, __func__, "1", LOGLEVEL); - sem_wait(job_clone_semaphore); + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); return(NULL); } mutex_mgr array_mgr(pa->ai_mutex, true); mutex_mgr template_job_mgr(template_job->ji_mutex,true); + if (template_job->ji_routed == false) + { + // Do not create the array sub-jobs until the job has been routed + set_task(WORK_Timed, time(NULL) + ARRAY_ROUTING_RETRY_SECONDS, job_clone_task_wrapper, strdup(jobid), FALSE); + free(jobid); + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + return(NULL); + } + free(jobid); // get the adjusted path_jobs path @@ -1437,6 +1448,7 @@ void *job_clone_wt( if (rc == FATAL_ERROR) { + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); return(NULL); } @@ -1450,8 +1462,7 @@ void *job_clone_wt( perform_array_postprocessing(pa); - unlock_ai_mutex(pa, __func__, "3", LOGLEVEL); - sem_wait(job_clone_semaphore); + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); return(NULL); } /* END job_clone_wt */ diff --git a/src/server/job_route.c b/src/server/job_route.c index 7e7d4cecc4..a049421b5b 100644 --- a/src/server/job_route.c +++ b/src/server/job_route.c @@ -248,6 +248,8 @@ int default_router( /* set time to retry job */ jobp->ji_qs.ji_un.ji_routet.ji_rteretry = retry_time; + if (jobp->ji_is_array_template) + jobp->ji_routed = false; return(0); } @@ -273,6 +275,8 @@ int default_router( case ROUTE_DEFERRED: /* deferred */ + jobp->ji_routed = true; + return(0); /*NOTREACHED*/ @@ -493,6 +497,48 @@ int reroute_job( +/* + * handle_rerouting() + * + */ + +int handle_rerouting( + + job *pjob, + pbs_queue *pque, + char *queue_name) + + { + int rc = PBSE_NONE; + + /* We only want to try if routing has been tried at least once - this is to let + * req_commit have the first crack at routing always. */ + if (pjob->ji_commit_done == 0) /* when req_commit is done it will set ji_commit_done to 1 */ + { + unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL); + return(rc); + } + + /* queue must be unlocked when calling reroute_job */ + unlock_queue(pque, __func__, NULL, 10); + reroute_job(pjob); + unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL); + + /* need to relock queue when we go to call next_job */ + pque = find_queuebyname(queue_name); + if (pque == NULL) + { + char log_buf[LOCAL_LOG_BUF_SIZE]; + sprintf(log_buf, "Could not find queue %s", queue_name); + log_err(-1, __func__, log_buf); + rc = -1; + } + + return(rc); + } // END handle_rerouting() + + + /* * queue_route - route any "ready" jobs in a specific queue @@ -557,33 +603,34 @@ void *queue_route( while ((pjob = next_job(pque->qu_jobs,iter)) != NULL) { - /* We only want to try if routing has been tried at least once - this is to let - * req_commit have the first crack at routing always. */ - - if (pjob->ji_commit_done == 0) /* when req_commit is done it will set ji_commit_done to 1 */ + if (handle_rerouting(pjob, pque, queue_name) != PBSE_NONE) { - unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL); - continue; + free(queue_name); + delete iter; + pthread_mutex_unlock(reroute_job_mutex); + return(NULL); } + } + + delete iter; - /* queue must be unlocked when calling reroute_job */ - que_mutex.unlock(); - reroute_job(pjob); - unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL); + // Now try an array summary jobs that haven't been routed + pque->qu_jobs_array_sum->lock(); + iter = pque->qu_jobs_array_sum->get_iterator(); + pque->qu_jobs_array_sum->unlock(); - /* need to relock queue when we go to call next_job */ - pque = find_queuebyname(queue_name); - if (pque == NULL) + while ((pjob = next_job(pque->qu_jobs_array_sum, iter)) != NULL) + { + if (pjob->ji_is_array_template == false) + continue; + + if (handle_rerouting(pjob, pque, queue_name) != PBSE_NONE) { - sprintf(log_buf, "Could not find queue %s", queue_name); - log_err(-1, __func__, log_buf); free(queue_name); delete iter; pthread_mutex_unlock(reroute_job_mutex); return(NULL); } - - que_mutex.mark_as_locked(); } /* we come out of the while loop with the queue locked. diff --git a/src/server/pbsd_main.c b/src/server/pbsd_main.c index 530abd7458..c6db8b05b0 100644 --- a/src/server/pbsd_main.c +++ b/src/server/pbsd_main.c @@ -318,7 +318,6 @@ int MultiMomMode = 0; int allow_any_mom = FALSE; int array_259_upgrade = FALSE; -sem_t *job_clone_semaphore; /* used to track the number of job_clone_wt requests are outstanding */ acl_special limited_acls; authorized_hosts auth_hosts; @@ -1464,25 +1463,6 @@ void main_loop(void) track_save(NULL); /* save tracking data */ - /* let any array jobs that might still be cloning finish */ - int sem_val; - do - { - int rc; - - rc = sem_getvalue(job_clone_semaphore, &sem_val); - if (rc != 0) - { - sprintf(log_buf, "failed to get job_clone_semaphore value"); - log_err(-1, __func__, log_buf); - break; - } - - if (sem_val > 0) - sleep(1); - }while(sem_val > 0); - - alljobs.lock(); iter = alljobs.get_iterator(); alljobs.unlock(); @@ -1610,7 +1590,6 @@ int main( { int i; - int rc; int local_errno = 0; char lockfile[MAXPATHLEN + 1]; char EMsg[MAX_LINE]; @@ -1684,21 +1663,6 @@ int main( if (server_name_file_port != 0) pbs_server_port_dis = server_name_file_port; - /* initialize job clone semaphore so we can track the number of outstanding - job array creation requests */ - job_clone_semaphore = (sem_t *)malloc(sizeof(sem_t)); - if (job_clone_semaphore == NULL) - { - perror("failed to allocate memory for job_clone_semaphore"); - exit(1); - } - rc = sem_init(job_clone_semaphore, 1 /* share */, 0); - if (rc != 0) - { - perror("failed to initialize job clone semaphore"); - exit(1); - } - strcpy(pbs_server_name, server_name); /* The following port numbers might have been initialized in set_globals_from_environment() above. */ diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index f28fbf3706..bbf445fb39 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -552,7 +552,7 @@ int svr_enquejob( if (!pjob->ji_is_array_template) { rc = insert_into_alljobs_by_rank(pque->qu_jobs, pjob, job_id); - + if ((rc == ALREADY_IN_LIST) || (rc == PBSE_JOBNOTFOUND)) { @@ -2480,9 +2480,10 @@ int check_for_complete_req_and_limits(struct pbs_queue *const pque, job *pjob) static int check_queue_job_limit( - struct job *const pjob, - struct pbs_queue *const pque, - char *const EMsg) + struct job *const pjob, + struct pbs_queue *const pque, + char *const EMsg) + { int return_code = PBSE_NONE; /* Optimistic assumption */ int array_jobs = 0; From 8726c267c63c6f941794f7995465fd9c868c39ae Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 3 Apr 2017 16:50:04 -0600 Subject: [PATCH 212/312] TRQ-3934. Add mem_per_task and swap_per_task, and fix some confusion around this. --- src/include/req.hpp | 2 + src/lib/Libattr/complete_req.cpp | 11 ++- src/lib/Libattr/req.cpp | 120 ++++++++++++++++++----- src/test/attr_fn_complete_req/test_uut.c | 6 +- src/test/req/test_uut.c | 71 ++++++++------ 5 files changed, 144 insertions(+), 66 deletions(-) diff --git a/src/include/req.hpp b/src/include/req.hpp index ba7c634126..368c19131e 100644 --- a/src/include/req.hpp +++ b/src/include/req.hpp @@ -117,7 +117,9 @@ class req { int execution_slots; unsigned long mem; + unsigned long mem_per_task; unsigned long swap; + unsigned long swap_per_task; unsigned long disk; int nodes; int socket; diff --git a/src/lib/Libattr/complete_req.cpp b/src/lib/Libattr/complete_req.cpp index ccacfdb15f..540daa94cd 100644 --- a/src/lib/Libattr/complete_req.cpp +++ b/src/lib/Libattr/complete_req.cpp @@ -143,10 +143,12 @@ complete_req::complete_req( } } + if (task_count == 0) + task_count = 1; + // Set mem_per_task and swap_per_task from mem_values unsigned long long mem_per_task = mem_values[_MEM_]; - if (task_count != 0) - mem_per_task /= task_count; + mem_per_task /= task_count; if (this->reqs.size() > 0) { @@ -166,8 +168,7 @@ complete_req::complete_req( } unsigned long long vmem_per_task = mem_values[_VMEM_]; - if (task_count != 0) - vmem_per_task /= task_count; + vmem_per_task /= task_count; if (this->reqs.size() > 0) { @@ -191,7 +192,7 @@ complete_req::complete_req( // Handle the case where no -lnodes request was made // Distribute the memory across the tasks as -l memory is per job req r; - if (task_count != 0) + if (task_count > 1) { r.set_task_count(task_count); r.set_placement_type(place_legacy); diff --git a/src/lib/Libattr/req.cpp b/src/lib/Libattr/req.cpp index ec98e8089b..37e343bfce 100644 --- a/src/lib/Libattr/req.cpp +++ b/src/lib/Libattr/req.cpp @@ -34,7 +34,7 @@ const char *place_thread = "thread"; const char *place_legacy = "legacy"; const char *place_legacy2 = "legacy2"; -req::req() : execution_slots(1), mem(0), swap(0), disk(0), nodes(0), +req::req() : execution_slots(1), mem(0), mem_per_task(0), swap(0), swap_per_task(0), disk(0), nodes(0), socket(0), numa_nodes(0), cores(0), threads(0), thread_usage_policy(ALLOW_THREADS), thread_usage_str(allow_threads), gpus(0), mics(0), maxtpn(0), gres(), os(), arch(), node_access_policy(), features(), placement_str(), req_attr(), gpu_mode(), @@ -48,12 +48,12 @@ req::req() : execution_slots(1), mem(0), swap(0), disk(0), nodes(0), req::req( - char *work_str) : execution_slots(1), mem(0), swap(0), disk(0), nodes(0), socket(0), - numa_nodes(0), cores(0), threads(0), thread_usage_policy(ALLOW_THREADS), - thread_usage_str(allow_threads), gpus(0), mics(0), maxtpn(0), gres(), - os(), arch(), node_access_policy(), features(), placement_str(), gpu_mode(), - task_count(1), pack(false), single_job_access(false), per_task_cgroup(-1), - index(0), hostlist(), task_allocations() + char *work_str) : execution_slots(1), mem(0), mem_per_task(0), swap(0), swap_per_task(0), disk(0), + nodes(0), socket(0), numa_nodes(0), cores(0), threads(0), + thread_usage_policy(ALLOW_THREADS), thread_usage_str(allow_threads), gpus(0), + mics(0), maxtpn(0), gres(), os(), arch(), node_access_policy(), features(), + placement_str(), gpu_mode(), task_count(1), pack(false), single_job_access(false), + per_task_cgroup(-1), index(0), hostlist(), task_allocations() { char *ptr = work_str; @@ -110,7 +110,9 @@ req::req( const req &other) : execution_slots(other.execution_slots), mem(other.mem), + mem_per_task(other.mem_per_task), swap(other.swap), + swap_per_task(other.swap_per_task), disk(other.disk), nodes(other.nodes), socket(other.socket), @@ -535,7 +537,12 @@ int req::set_name_value_pair( rc = parse_positive_integer(value, this->execution_slots); } else if (!strcmp(name, "memory")) - rc = read_mem_value(value, this->mem); + { + rc = read_mem_value(value, this->mem_per_task); + + if (rc == PBSE_NONE) + this->mem = this->mem_per_task * this->task_count; + } else if (!strcmp(name, "gpus")) rc = parse_positive_integer(value, this->gpus); else if (!strcmp(name, "mics")) @@ -560,7 +567,12 @@ int req::set_name_value_pair( else if (!strcmp(name, "reqattr")) this->req_attr = value; else if (!strcmp(name, "swap")) - rc = read_mem_value(value, this->swap); + { + rc = read_mem_value(value, this->swap_per_task); + + if (rc == PBSE_NONE) + this->swap = this->swap_per_task * this->task_count; + } else return(PBSE_BAD_PARAMETER); @@ -1111,15 +1123,16 @@ int req::set_from_submission_string( req::req( - const std::string &resource_request) : execution_slots(1), mem(0), swap(0), disk(0), - nodes(0), socket(0), numa_nodes(0), - cores(0), threads(0), thread_usage_policy(ALLOW_THREADS), - thread_usage_str(allow_threads), gpus(0), mics(0), - maxtpn(0), gres(), os(), arch(), node_access_policy(), - features(), placement_str(), req_attr(), gpu_mode(), - task_count(1), pack(false), single_job_access(false), - per_task_cgroup(-1), index(-1), hostlist(), - task_allocations() + const std::string &resource_request) : execution_slots(1), mem(0), mem_per_task(0), swap(0), + swap_per_task(0), disk(0), nodes(0), socket(0), + numa_nodes(0), cores(0), threads(0), + thread_usage_policy(ALLOW_THREADS), + thread_usage_str(allow_threads), gpus(0), mics(0), + maxtpn(0), gres(), os(), arch(), node_access_policy(), + features(), placement_str(), req_attr(), gpu_mode(), + task_count(1), pack(false), single_job_access(false), + per_task_cgroup(-1), index(-1), hostlist(), + task_allocations() { char *work_str = strdup(resource_request.c_str()); @@ -1146,7 +1159,9 @@ req &req::operator =( this->execution_slots = other.execution_slots; this->mem = other.mem; + this->mem_per_task = other.mem_per_task; this->swap = other.swap; + this->swap_per_task = other.swap_per_task; this->disk = other.disk; this->nodes = other.nodes; this->socket = other.socket; @@ -1217,12 +1232,24 @@ void req::toString( str += buf; } + if (this->mem_per_task != 0) + { + snprintf(buf, sizeof(buf), " mem_per_task: %lukb\n", this->mem_per_task); + str += buf; + } + if (this->swap != 0) { snprintf(buf, sizeof(buf), " swap: %lukb\n", this->swap); str += buf; } + if (this->swap_per_task != 0) + { + snprintf(buf, sizeof(buf), " swap_per_task: %lukb\n", this->swap_per_task); + str += buf; + } + if (this->disk != 0) { snprintf(buf, sizeof(buf), " disk: %lukb\n", this->disk); @@ -1401,6 +1428,14 @@ void req::get_values( names.push_back(buf); snprintf(buf, sizeof(buf), "%lukb", this->mem); values.push_back(buf); + + if (this->mem_per_task != 0) + { + snprintf(buf, sizeof(buf), "memory_per_task.%d", this->index); + names.push_back(buf); + snprintf(buf, sizeof(buf), "%lukb", this->mem_per_task); + values.push_back(buf); + } } if (this->swap != 0) @@ -1409,6 +1444,14 @@ void req::get_values( names.push_back(buf); snprintf(buf, sizeof(buf), "%lukb", this->swap); values.push_back(buf); + + if (this->swap_per_task != 0) + { + snprintf(buf, sizeof(buf), "swap_per_task.%d", this->index); + names.push_back(buf); + snprintf(buf, sizeof(buf), "%lukb", this->swap_per_task); + values.push_back(buf); + } } if (this->disk != 0) @@ -1666,7 +1709,9 @@ char *capture_until_newline_and_advance( * task count: * [lprocs: ] * [mem: kb] + * [mem_per_task: kb] * [swap: kb] + * [swap_per_task: kb] * [disk: kb] * [socket: ] * [numa chips: ] @@ -1738,6 +1783,17 @@ void req::set_from_string( // found mem current += 5; // move past 'mem: ' this->mem = strtoll(current, ¤t, 10); + this->mem_per_task = this->mem / this->task_count; + current += 2; // move past 'kb' + + move_past_whitespace(¤t); + } + + if (!strncmp(current, "mem_per_task", 12)) + { + // found mem_per_task + current += 14; // move past 'mem_per_task: ' + this->mem_per_task = strtoll(current, ¤t, 10); current += 2; // move past 'kb' move_past_whitespace(¤t); @@ -1748,6 +1804,17 @@ void req::set_from_string( // found swap current += 6; // move past 'swap: ' this->swap = strtoll(current, ¤t, 10); + this->swap_per_task = this->swap / this->task_count; + current += 2; // move past kb + + move_past_whitespace(¤t); + } + + if (!strncmp(current, "swap_per_task", 13)) + { + // found swap + current += 15; // move past 'swap_per_task: ' + this->swap = strtoll(current, ¤t, 10); current += 2; // move past kb move_past_whitespace(¤t); @@ -2032,6 +2099,8 @@ int req::set_value( { if ((rc = read_mem_value(value, this->mem)) != PBSE_NONE) return(rc); + else + this->mem_per_task = this->mem / this->task_count; } } else if (!strncmp(name, "swap", 4)) @@ -2041,6 +2110,8 @@ int req::set_value( { if ((rc = read_mem_value(value, this->swap)) != PBSE_NONE) return(rc); + else + this->swap_per_task = this->swap / this->task_count; } } else if (!strncmp(name, "disk", 4)) @@ -2479,7 +2550,7 @@ int req::get_task_allocation( unsigned long long req::get_swap_per_task() { - return(this->swap); + return(this->swap_per_task); } @@ -2493,7 +2564,7 @@ unsigned long long req::get_swap_per_task() unsigned long long req::get_memory_per_task() { - return(this->mem); + return(this->mem_per_task); } /* @@ -2509,9 +2580,7 @@ unsigned long long req::get_swap_for_host( { int num_tasks = this->get_num_tasks_for_host(host); - unsigned long long swap = this->swap * num_tasks; - - return(swap); + return(this->swap_per_task * num_tasks); } // END get_swap_for_host() @@ -2529,9 +2598,7 @@ unsigned long long req::get_memory_for_host( { int num_tasks = this->get_num_tasks_for_host(host); - unsigned long long mem = (this->mem * num_tasks) / this->task_count; - - return(mem); + return(this->mem_per_task * num_tasks); } // END get_memory_for_host() @@ -2583,6 +2650,7 @@ void req::set_memory( { this->mem = mem; + this->mem_per_task = this->mem / this->task_count; } void req::set_swap( diff --git a/src/test/attr_fn_complete_req/test_uut.c b/src/test/attr_fn_complete_req/test_uut.c index 1561defa40..3899998498 100644 --- a/src/test/attr_fn_complete_req/test_uut.c +++ b/src/test/attr_fn_complete_req/test_uut.c @@ -66,12 +66,12 @@ START_TEST(test_decode_encode_complete_req) called_encode = 0; fail_unless(encode_complete_req(&pattr, NULL, NULL, NULL, 0, 0) == PBSE_NONE); - // we set 2 attributes + there are 2 that should go by default - fail_unless(called_encode == 4, "encoded %d", called_encode); + // we set 2 attributes + there are 2 that should go by default + mem_per_task + fail_unless(called_encode == 5, "encoded %d", called_encode); pattr.at_val.at_ptr = NULL; fail_unless(encode_complete_req(&pattr, NULL, NULL, NULL, 0, 0) == PBSE_NONE); // make sure we didn't encode anything new - fail_unless(called_encode == 4); + fail_unless(called_encode == 5); } END_TEST diff --git a/src/test/req/test_uut.c b/src/test/req/test_uut.c index 47cdeaa60e..869b73774e 100644 --- a/src/test/req/test_uut.c +++ b/src/test/req/test_uut.c @@ -145,13 +145,13 @@ START_TEST(test_get_set_values) req r; r.set_value("index", "0", false); + r.set_value("task_count", "5", false); r.set_value("lprocs", "all", false); r.set_value("memory", "1024mb", false); r.set_value("swap", "1024kb", false); r.set_value("disk", "10000000kb", false); r.set_value("socket", "1", false); r.set_value("gpus", "2", false); - r.set_value("task_count", "5", false); r.set_value("gpu_mode", "exclusive_thread", false); r.set_value("mics", "1", false); r.set_value("thread_usage_policy", "use threads", false); @@ -173,38 +173,42 @@ START_TEST(test_get_set_values) fail_unless(names[0] == "task_count.0"); fail_unless(names[1] == "lprocs.0"); fail_unless(names[2] == "memory.0"); - fail_unless(names[3] == "swap.0"); - fail_unless(names[4] == "disk.0"); - fail_unless(names[5] == "socket.0"); - fail_unless(names[6] == "gpus.0"); - fail_unless(names[7] == "gpu_mode.0"); - fail_unless(names[8] == "mics.0"); - fail_unless(names[9] == "thread_usage_policy.0", names[9].c_str()); - fail_unless(names[10] == "reqattr.0"); - fail_unless(names[11] == "gres.0"); - fail_unless(names[12] == "opsys.0"); - fail_unless(names[13] == "arch.0"); - fail_unless(names[14] == "features.0", names[14].c_str()); - fail_unless(names[15] == "single_job_access.0"); - fail_unless(names[16] == "hostlist.0", names[16].c_str()); + fail_unless(names[3] == "memory_per_task.0"); + fail_unless(names[4] == "swap.0"); + fail_unless(names[5] == "swap_per_task.0"); + fail_unless(names[6] == "disk.0"); + fail_unless(names[7] == "socket.0"); + fail_unless(names[8] == "gpus.0"); + fail_unless(names[9] == "gpu_mode.0"); + fail_unless(names[10] == "mics.0"); + fail_unless(names[11] == "thread_usage_policy.0", names[9].c_str()); + fail_unless(names[12] == "reqattr.0"); + fail_unless(names[13] == "gres.0"); + fail_unless(names[14] == "opsys.0"); + fail_unless(names[15] == "arch.0"); + fail_unless(names[16] == "features.0", names[14].c_str()); + fail_unless(names[17] == "single_job_access.0"); + fail_unless(names[18] == "hostlist.0", names[16].c_str()); fail_unless(values[0] == "5"); fail_unless(values[1] == "all"); fail_unless(values[2] == "1048576kb", "value: %s", values[2].c_str()); - fail_unless(values[3] == "1024kb"); - fail_unless(values[4] == "10000000kb"); - fail_unless(values[5] == "1"); - fail_unless(values[6] == "2"); - fail_unless(values[7] == "exclusive_thread"); - fail_unless(values[8] == "1"); - fail_unless(values[9] == "use threads"); - fail_unless(values[10] == "matlab>7"); - fail_unless(values[11] == "gresA"); - fail_unless(values[12] == "ubuntu"); - fail_unless(values[13] == "64bit"); - fail_unless(values[14] == "fast"); - fail_unless(values[15] == "true"); - fail_unless(values[16] == "napali:ppn=32"); + fail_unless(values[3] == "209715kb", "value: %s", values[3].c_str()); + fail_unless(values[4] == "1024kb"); + fail_unless(values[5] == "204kb"); + fail_unless(values[6] == "10000000kb"); + fail_unless(values[7] == "1"); + fail_unless(values[8] == "2"); + fail_unless(values[9] == "exclusive_thread"); + fail_unless(values[10] == "1"); + fail_unless(values[11] == "use threads"); + fail_unless(values[12] == "matlab>7"); + fail_unless(values[13] == "gresA"); + fail_unless(values[14] == "ubuntu"); + fail_unless(values[15] == "64bit"); + fail_unless(values[16] == "fast"); + fail_unless(values[17] == "true"); + fail_unless(values[18] == "napali:ppn=32"); req r2; r2.set_value("lprocs", "2", false); @@ -367,7 +371,8 @@ START_TEST(test_constructors) fail_unless(r2.getGres() == "matlab=1"); fail_unless(r2.getDisk() == 0); fail_unless(r2.getSwap() == 0); - fail_unless(r2.getMemory() == 12 * 1024 * 1024); + fail_unless(r2.getMemory() == 12 * 1024 * 1024 * 5); + fail_unless(r2.get_memory_per_task() == 12 * 1024 * 1024); fail_unless(r2.getExecutionSlots() == 4); fail_unless(r2.getFeatures() == "fast", "features '%s'", r2.getFeatures().c_str()); @@ -379,7 +384,8 @@ START_TEST(test_constructors) fail_unless(copy_r2.getGres() == "matlab=1"); fail_unless(copy_r2.getDisk() == 0); fail_unless(copy_r2.getSwap() == 0); - fail_unless(copy_r2.getMemory() == 12 * 1024 * 1024); + fail_unless(copy_r2.getMemory() == 12 * 1024 * 1024 * 5); + fail_unless(copy_r2.get_memory_per_task() == 12 * 1024 * 1024); fail_unless(copy_r2.getExecutionSlots() == 4); fail_unless(copy_r2.getFeatures() == "fast", "features '%s'", copy_r2.getFeatures().c_str()); @@ -520,6 +526,7 @@ START_TEST(test_set_from_string) fail_unless(r.getFeatures() == "fast"); fail_unless(r.getExecutionSlots() == ALL_EXECUTION_SLOTS); fail_unless(r.getMemory() == 10000); + fail_unless(r.get_memory_per_task() == 10000 / 10, "%d per task", r.get_memory_per_task()); fail_unless(r.getSwap() == 1024); fail_unless(r.getDisk() == 10000000); fail_unless(r.getTaskCount() == 10); @@ -582,10 +589,10 @@ START_TEST(test_get_memory_for_host) unsigned long mem; r.set_value("index", "0", false); + r.set_value("task_count", "5", false); r.set_value("lprocs", "all", false); r.set_value("memory", "1024kb", false); r.set_value("swap", "1024kb", false); - r.set_value("task_count", "5", false); r.set_value("thread_usage_policy", "use threads", false); r.set_value("hostlist", "napali:ppn=32", false); From 9e53c594d6e1d5faf084916118a398468af9825b Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 3 Apr 2017 16:56:09 -0600 Subject: [PATCH 213/312] TRQ-3901. Clarify varattr time and how it's applied. --- src/include/mom_config.h | 1 + src/resmom/mom_main.c | 7 +++++++ src/resmom/parse_config.c | 4 ++-- src/test/mom_main/scaffolding.c | 2 ++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/include/mom_config.h b/src/include/mom_config.h index 8198ec3f6b..bdb2b2c7d3 100644 --- a/src/include/mom_config.h +++ b/src/include/mom_config.h @@ -116,6 +116,7 @@ extern char *auto_ideal_load; extern char *auto_max_load; extern int exec_with_exec; extern int ServerStatUpdateInterval; +extern int varattr_tv; extern char *AllocParCmd; extern char PBSNodeCheckPath[]; extern int PBSNodeCheckProlog; diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index d6cb27cbe5..c5afa90a7e 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -5142,6 +5142,13 @@ int setup_program_environment(void) exit(1); } + if(varattr_tv != -1 && varattr_tv < ServerStatUpdateInterval) + { + sprintf(log_buffer, + "WARNING: varattr time value of %i is less than server update interval of ($status_update_time) %i. Varattr will only be run after the server update interval. Suggest setting varattr time to a multiple of server update interval\n", + varattr_tv, ServerStatUpdateInterval); + log_record(PBSE_NONE,PBS_EVENTCLASS_SERVER,msg_daemonname,log_buffer); + } #ifdef PENABLE_LINUX26_CPUSETS rc = initialize_hwloc_topology(); diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index d65d946e41..bc972e8450 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -128,6 +128,7 @@ double cputfactor = 1.00; float ideal_load_val = -1.0; int exec_with_exec = 0; int ServerStatUpdateInterval = DEFAULT_SERVER_STAT_UPDATES; +int varattr_tv = -1; float max_load_val = -1.0; char *auto_ideal_load = NULL; char *auto_max_load = NULL; @@ -969,7 +970,7 @@ u_long setvarattr( ptr = value; pva->va_ttl = strtol(ptr, NULL, 10); - + varattr_tv = pva->va_ttl; /* step forward to end of TTL */ while ((!isspace(*ptr)) && @@ -3275,7 +3276,6 @@ const char *reqgres( strncat(GResBuf, tmpLine, (sizeof(GResBuf) - strlen(GResBuf) - 1)); } /* END for (cp) */ - return(GResBuf); } /* END reqgres() */ diff --git a/src/test/mom_main/scaffolding.c b/src/test/mom_main/scaffolding.c index 43ed5b5e07..b02611263b 100644 --- a/src/test/mom_main/scaffolding.c +++ b/src/test/mom_main/scaffolding.c @@ -89,6 +89,7 @@ struct config *config_array = NULL; char **maskclient = NULL; /* wildcard connections */ char PBSNodeCheckPath[MAXLINE]; int CheckPollTime; +int varattr_tv; char rcp_args[MAXPATHLEN]; long log_file_max_size; char mom_host[PBS_MAXHOSTNAME + 1]; @@ -1141,3 +1142,4 @@ int read_all_devices() { return(PBSE_NONE); } + From 76ee091ee5db430e6907584e43c470373e9c312c Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 3 Apr 2017 16:58:39 -0600 Subject: [PATCH 214/312] TRQ-3248. Add the array id to error messages for array recovery. --- src/server/array_upgrade.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/server/array_upgrade.c b/src/server/array_upgrade.c index 92849478bd..458eb9d417 100644 --- a/src/server/array_upgrade.c +++ b/src/server/array_upgrade.c @@ -43,6 +43,7 @@ int array_upgrade( { char log_buf[LOCAL_LOG_BUF_SIZE]; + std::string array_id(pa->ai_qs.parent_id); /* reset the file descriptor */ if (lseek(fds, 0, SEEK_SET) != 0) @@ -72,9 +73,9 @@ int array_upgrade( about the incompatibility and inability to upgrade, print a quick and dirty error message and exit */ - sprintf(log_buf, "WARNING, unable to upgrade job array from version %d\n" + sprintf(log_buf, "WARNING, unable to upgrade job array %s from version %d\n" "structs from versions prior to 2.5.0.\n" - "Please downgrade TORQUE and upgrade once no job arrays are queued\n", version); + "Please downgrade TORQUE and upgrade once no job arrays are queued\n",array_id.c_str(), version); log_err(-1, "array_upgrade", log_buf); exit(1); } @@ -93,8 +94,8 @@ int array_upgrade( * know and ignore the array, but don't halt the server. */ sprintf(log_buf, - "WARNING, unable to upgrade job array from version %d. Job array files most likely corrupted.\n" - "Please examine the files at %s for file integrity\n", version, path_arrays); + "WARNING, unable to upgrade job array %s from version %d. Job array files most likely corrupted.\n" + "Please examine the files at %s for file integrity\n", array_id.c_str(), version, path_arrays); log_err(-1, "array_upgrade", log_buf); return 1; } From c64dc07a66b8a76717b7688aa6be6a4c3a08695d Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 4 Apr 2017 10:52:10 -0600 Subject: [PATCH 215/312] TRQ-3934. Make sure task_count is never 0. --- src/lib/Libattr/req.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Libattr/req.cpp b/src/lib/Libattr/req.cpp index 37e343bfce..21e5fe8d5e 100644 --- a/src/lib/Libattr/req.cpp +++ b/src/lib/Libattr/req.cpp @@ -1757,7 +1757,8 @@ void req::set_from_string( } current += 2; // move past the ': ' - this->task_count = strtol(current, ¤t, 10); + if ((this->task_count = strtol(current, ¤t, 10)) == 0) + this->task_count = 1; while (is_whitespace(*current)) current++; From df784e2ec09a938a73ab9c0e9992a95c10edc3e5 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 6 Apr 2017 16:14:57 -0600 Subject: [PATCH 216/312] TRQ-3807 - change default sendmail path for RPMs --- buildutils/torque.adaptive.spec.el6 | 2 +- buildutils/torque.adaptive.spec.el7 | 2 +- buildutils/torque.adaptive.spec.sles12 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildutils/torque.adaptive.spec.el6 b/buildutils/torque.adaptive.spec.el6 index 32dd2f847a..012f481381 100644 --- a/buildutils/torque.adaptive.spec.el6 +++ b/buildutils/torque.adaptive.spec.el6 @@ -351,7 +351,7 @@ ### Macro variables %{!?torque_user:%global torque_user root} %{!?torque_home:%global torque_home %{_var}/spool/%{project_name}} -%{!?sendmail_path:%global sendmail_path %{_sbindir}/sendmail} +%{!?sendmail_path:%global sendmail_path /usr/sbin/sendmail} # This ensures that the debugging symbols are not stripped %define __os_install_post /usr/lib/rpm/brp-compress diff --git a/buildutils/torque.adaptive.spec.el7 b/buildutils/torque.adaptive.spec.el7 index 998aab355b..344bc13523 100644 --- a/buildutils/torque.adaptive.spec.el7 +++ b/buildutils/torque.adaptive.spec.el7 @@ -351,7 +351,7 @@ ### Macro variables %{!?torque_user:%global torque_user root} %{!?torque_home:%global torque_home %{_var}/spool/%{project_name}} -%{!?sendmail_path:%global sendmail_path %{_sbindir}/sendmail} +%{!?sendmail_path:%global sendmail_path /usr/sbin/sendmail} # This ensures that the debugging symbols are not stripped %define __os_install_post /usr/lib/rpm/brp-compress diff --git a/buildutils/torque.adaptive.spec.sles12 b/buildutils/torque.adaptive.spec.sles12 index 9eeaa7d802..63215bc56f 100644 --- a/buildutils/torque.adaptive.spec.sles12 +++ b/buildutils/torque.adaptive.spec.sles12 @@ -351,7 +351,7 @@ ### Macro variables %{!?torque_user:%global torque_user root} %{!?torque_home:%global torque_home %{_var}/spool/%{project_name}} -%{!?sendmail_path:%global sendmail_path %{_sbindir}/sendmail} +%{!?sendmail_path:%global sendmail_path /usr/sbin/sendmail} # This ensures that the debugging symbols are not stripped %define __os_install_post /usr/lib/rpm/brp-compress From 49758a6fcad2285a13427e4fe1fd2133a8322a67 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 7 Apr 2017 08:42:52 -0600 Subject: [PATCH 217/312] TRQ-3877 - fix rpmbuild problem --- buildutils/torque.spec.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildutils/torque.spec.in b/buildutils/torque.spec.in index 31f2aae4ce..79f0b7d40e 100644 --- a/buildutils/torque.spec.in +++ b/buildutils/torque.spec.in @@ -387,6 +387,9 @@ test "x$RPM_BUILD_ROOT" != "x/" && rm -rf $RPM_BUILD_ROOT %doc INSTALL INSTALL.GNU CHANGELOG PBS_License.txt README.* Release_Notes src/pam/README.pam %doc doc/READ_ME doc/doc_fonts doc/soelim.c doc/ers %config(noreplace) %{torque_home}/pbs_environment +%config(noreplace) /etc/ld.so.conf.d/torque.conf +%config(noreplace) /etc/profile.d/torque.sh +%config(noreplace) /etc/profile.d/torque.csh %config(noreplace) %{torque_home}/server_name %if %{?_unitdir:1}0 %{_unitdir}/trqauthd.service From a591051d916bef676d481c1eef3133ceea4a9abb Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 7 Apr 2017 13:17:30 -0600 Subject: [PATCH 218/312] Fix a spelling error in an error message. --- src/include/pbs_error_db.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/pbs_error_db.h b/src/include/pbs_error_db.h index 919b7f70e1..56f363adf8 100644 --- a/src/include/pbs_error_db.h +++ b/src/include/pbs_error_db.h @@ -315,8 +315,8 @@ PbsErrRm(PBSE_RMBADPARAM, (char *)"parameter could not be used") PbsErrRm(PBSE_RMNOPARAM, (char *)"a parameter needed did not exist") /* something specified didn't exist */ PbsErrRm(PBSE_RMEXIST, (char *)"something specified didn't exist") -/* a system error occured */ -PbsErrRm(PBSE_RMSYSTEM, (char *)"a system error occured") +/* a system error occurred */ +PbsErrRm(PBSE_RMSYSTEM, (char *)"a system error occurred") /* only part of reservation made */ PbsErrRm(PBSE_RMPART, (char *)"only part of reservation made") /* pbs rm errors ceiling (max_rm_err + 1) */ From 3bd4f4d268f3195fcb7b544e2c1ade2a1edabb6a Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 7 Apr 2017 14:04:45 -0600 Subject: [PATCH 219/312] Fix another spelling error in an error message. --- src/include/pbs_error_db.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/pbs_error_db.h b/src/include/pbs_error_db.h index 56f363adf8..b531953c3e 100644 --- a/src/include/pbs_error_db.h +++ b/src/include/pbs_error_db.h @@ -212,7 +212,7 @@ PbsErrClient(PBSE_NOJOBARRAYS, (char *)"Queue does not allow job arrays") /* 15090 */ PbsErrClient(PBSE_RELAYED_TO_MOM, (char *)"request was relayed to a MOM") /* */ PbsErrClient(PBSE_MEM_MALLOC, (char *)"Error allocating memory - out of memory") -PbsErrClient(PBSE_MUTEX, (char *)"Error allocating controling mutex (lock/unlock)") +PbsErrClient(PBSE_MUTEX, (char *)"Error allocating controlling mutex (lock/unlock)") PbsErrClient(PBSE_THREADATTR, (char *)"Error setting thread attributes") PbsErrClient(PBSE_THREAD, (char *)"Error creating thread") PbsErrClient(PBSE_SELECT, (char *)"Error in socket select") From 4de960f133a47929332202b3e3066ee29e6b3225 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 10 Apr 2017 15:31:31 -0600 Subject: [PATCH 220/312] TRQ-3766 - clean up nvml logging --- src/resmom/nvidia.c | 90 +++++++++++++++++------------------ src/test/nvidia/scaffolding.c | 18 ++++++- src/test/nvidia/test_uut.c | 24 ++++++++++ 3 files changed, 83 insertions(+), 49 deletions(-) diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index 3a0c630919..f7c8a9f897 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -1405,6 +1405,7 @@ void generate_server_gpustatus_nvml( nvmlUtilization_t util_info; unsigned long long ecc_counts; char tmpbuf[1024+1]; + char log_buf[LOG_BUF_SIZE]; if (!init_nvidia_nvml(global_gpu_count)) return; @@ -1442,7 +1443,9 @@ void generate_server_gpustatus_nvml( } else { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlSystemGetDriverVersion() called from %s", __func__); + log_nvml_error(rc, NULL, log_buf); } #ifndef NUMA_SUPPORT @@ -1451,7 +1454,9 @@ void generate_server_gpustatus_nvml( rc = nvmlDeviceGetCount(&device_count); if (rc != NVML_SUCCESS) { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetCount() called from %s", __func__); + log_nvml_error(rc, NULL, log_buf); shut_nvidia_nvml(); return; } @@ -1469,7 +1474,9 @@ void generate_server_gpustatus_nvml( if (rc != NVML_SUCCESS) { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetHandleByIndex() called from %s (idx=%d)", __func__, idx); + log_nvml_error(rc, NULL, log_buf); continue; } @@ -1490,7 +1497,9 @@ void generate_server_gpustatus_nvml( } else { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetDisplayMode() called from %s (idx=%d)", __func__, idx); + log_nvml_error(rc, NULL, log_buf); gpu_status.push_back("gpu_display=Unknown"); } @@ -1515,7 +1524,9 @@ void generate_server_gpustatus_nvml( } else { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetPciInfo() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the product name */ @@ -1529,7 +1540,9 @@ void generate_server_gpustatus_nvml( } else { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetName() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the fan speed */ @@ -1542,11 +1555,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetFanSpeed() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the memory information */ @@ -1562,11 +1573,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetMemoryInfo() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the compute mode */ @@ -1612,11 +1621,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetComputeMode() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the utilization rates */ @@ -1633,11 +1640,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetComputeMode() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the ECC mode */ @@ -1652,11 +1657,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetEccMode() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the single bit ECC errors */ @@ -1671,11 +1674,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetTotalEccErrors() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the double bit ECC errors */ @@ -1690,11 +1691,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetTotalEccErrors() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the temperature */ @@ -1708,13 +1707,10 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetTemperature() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } - } shut_nvidia_nvml(); diff --git a/src/test/nvidia/scaffolding.c b/src/test/nvidia/scaffolding.c index 45ecc5d139..28cfe829e8 100644 --- a/src/test/nvidia/scaffolding.c +++ b/src/test/nvidia/scaffolding.c @@ -6,8 +6,10 @@ #include #include "log.h" #include "mcom.h" +#include char log_buffer[LOG_BUF_SIZE]; +nvmlReturn_t global_nvmlDeviceGetFanSpeed_rc = NVML_SUCCESS; int MXMLFromString( @@ -23,8 +25,6 @@ int MXMLFromString( int MXMLDestroyE(mxml_t **EP) {return(0); } -void log_err(int errnum, const char *routine, const char *text) {} - void log_ext(int errnum, const char *routine, const char *text, int severity) { } void log_event(int eventtype, int objclass, const char *objname, const char *text) {} @@ -32,3 +32,17 @@ void log_event(int eventtype, int objclass, const char *objname, const char *tex void send_update_soon() { return; } void get_device_indices(const char *device_str, std::vector &device_indices, const char *suffix) {} + +nvmlReturn_t nvmlDeviceGetFanSpeed(nvmlDevice_t device_hndl, unsigned int *i) + { + return(global_nvmlDeviceGetFanSpeed_rc); + } + +void log_err( + int errnum, /* I (errno or PBSErrno) */ + const char *routine, /* I */ + const char *text) /* I */ + + { + snprintf(log_buffer, sizeof(log_buffer), "%s %s", routine, text); + } diff --git a/src/test/nvidia/test_uut.c b/src/test/nvidia/test_uut.c index e6fff33605..b979031dd8 100644 --- a/src/test/nvidia/test_uut.c +++ b/src/test/nvidia/test_uut.c @@ -17,7 +17,10 @@ int global_gpu_count = 0; int time_now = 0; char mom_host[PBS_MAXHOSTNAME + 1]; +extern nvmlReturn_t global_nvmlDeviceGetFanSpeed_rc; + int get_nvml_version(); +void generate_server_gpustatus_nvml(std::vector &gpu_status); START_TEST(test_get_nvml_version) { @@ -32,6 +35,23 @@ START_TEST(test_get_nvml_version) } END_TEST +START_TEST(test_generate_server_gpustatus_nvml) + { + std::vector gpu_status; + + // force nvmlDeviceGetFanSpeed() to return error + global_nvmlDeviceGetFanSpeed_rc = NVML_ERROR_UNKNOWN; + LOGLEVEL = 2; + + generate_server_gpustatus_nvml(gpu_status); + + // expect log_buffer to hold something like + // "nvmlDeviceGetFanSpeed() called from generate_server_gpustatus_nvml (idx=0) Unknown error" + fail_unless((strcasestr(log_buffer, "nvmlDeviceGetFanSpeed() called from generate_server_gpustatus_nvml") != NULL) && + (strcasestr(log_buffer, "unknown error") != NULL)); + } +END_TEST + START_TEST(test_two) { } @@ -48,6 +68,10 @@ Suite *nvidia_suite(void) tcase_add_test(tc_core, test_get_nvml_version); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_generate_server_gpustatus_nvml"); + tcase_add_test(tc_core, test_generate_server_gpustatus_nvml); + suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_two"); tcase_add_test(tc_core, test_two); suite_add_tcase(s, tc_core); From 6ab8b280b31220b66398bf20f4caaa25f9308a15 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 11 Apr 2017 15:32:35 -0600 Subject: [PATCH 221/312] TRQ-3932. Fix a deadlock in my commit (the job was being left locked). --- src/server/job_route.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/server/job_route.c b/src/server/job_route.c index a049421b5b..1d85ea3435 100644 --- a/src/server/job_route.c +++ b/src/server/job_route.c @@ -500,6 +500,9 @@ int reroute_job( /* * handle_rerouting() * + * precond: pjob is locked + * postcond: pjob is unlocked + * */ int handle_rerouting( @@ -539,7 +542,6 @@ int handle_rerouting( - /* * queue_route - route any "ready" jobs in a specific queue * @@ -622,7 +624,10 @@ void *queue_route( while ((pjob = next_job(pque->qu_jobs_array_sum, iter)) != NULL) { if (pjob->ji_is_array_template == false) + { + unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL); continue; + } if (handle_rerouting(pjob, pque, queue_name) != PBSE_NONE) { From 7fb92fad3b29553142346c806aff8b0ed8aa3e70 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 11 Apr 2017 17:15:19 -0600 Subject: [PATCH 222/312] TRQ-3973. Make allow_node_submit compatible with FQDNs as well as short names. This is a new commit instead of cherry-picking due to functions being shuffled around. --- src/server/geteusernam.c | 90 +++++++++--------------------- src/test/geteusernam/scaffolding.c | 31 +++++++--- src/test/geteusernam/test_uut.c | 50 ++++++++++------- 3 files changed, 78 insertions(+), 93 deletions(-) diff --git a/src/server/geteusernam.c b/src/server/geteusernam.c index f744cfe979..ccc4daf94d 100644 --- a/src/server/geteusernam.c +++ b/src/server/geteusernam.c @@ -315,37 +315,6 @@ bool getegroup( } /* END getegroup() */ -/* - * get_user_host_from_user - * - * @param user_host - receives the host name associated with the user. - * @param user - user name with or without the host name. - * - */ - -void get_user_host_from_user( - - std::string &user_host, - const std::string user) - - { - char *ptr; - char *tmp_name = strdup(user.c_str()); - - user_host.clear(); - ptr = strchr(tmp_name, '@'); - if (ptr != NULL) - { - ptr++; - user_host = ptr; - } - - free(tmp_name); - - } - - - /* * Verifies that the job's user is acceptable @@ -362,14 +331,13 @@ bool is_user_allowed_to_submit_jobs( { char *orighost; - std::string user_host; /* this is a back up to orighost in case we need FQDN */ std::string user(pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str); std::size_t at_pos = user.find('@'); int rc = PBSE_NONE; bool ProxyAllowed = false; bool ProxyRequested = false; bool HostAllowed = false; - char *dptr; + std::string short_host; char log_buf[256]; @@ -386,8 +354,6 @@ bool is_user_allowed_to_submit_jobs( if (EMsg != NULL) EMsg[0] = '\0'; - get_user_host_from_user(user_host, user); - if (at_pos != std::string::npos) user.erase(at_pos); @@ -405,6 +371,13 @@ bool is_user_allowed_to_submit_jobs( return(false); } + short_host = orighost; + size_t pos = short_host.find('.'); + if (pos == std::string::npos) + short_host.clear(); + else + short_host.erase(pos); + /* check to see if we are allowing a proxy user to submit the job */ if ((server.sv_attr[SRV_ATR_AllowProxyUser].at_flags & ATR_VFLAG_SET) && \ (server.sv_attr[SRV_ATR_AllowProxyUser].at_val.at_bool == true)) @@ -432,28 +405,26 @@ bool is_user_allowed_to_submit_jobs( HostAllowed = true; } - /* make short host name */ - if ((dptr = strchr(orighost, '.')) != NULL) + if (HostAllowed == false) { - *dptr = '\0'; - } + bool allowed = false; - if ((HostAllowed == false) && - (is_permitted_by_node_submit(orighost, logging))) - { - /* job submitted from compute host, access allowed */ - if (dptr != NULL) - *dptr = '.'; - - if ((ProxyRequested == false) || - (ProxyAllowed == true)) + if (is_permitted_by_node_submit(orighost, logging) == false) { - return(true); + if (short_host.size() > 0) + allowed = is_permitted_by_node_submit(short_host.c_str(), logging); } + else + allowed = true; - /* host is fine, must validate proxy via ruserok() */ - - HostAllowed = false; + if (allowed == true) + { + if ((ProxyRequested == false) || + (ProxyAllowed == true)) + { + return(true); + } + } } if ((HostAllowed == false) && @@ -472,20 +443,14 @@ bool is_user_allowed_to_submit_jobs( int cmpRet = strcasecmp(testhost, orighost); if ((cmpRet != 0) && - (dptr != NULL)) + (short_host.size() > 0)) { - *dptr = '.'; - cmpRet = strcasecmp(testhost, orighost); - *dptr = '\0'; + cmpRet = strcasecmp(testhost, short_host.c_str()); } if (cmpRet == 0) { /* job submitted from host found in trusted submit host list, access allowed */ - - if (dptr != NULL) - *dptr = '.'; - if ((ProxyRequested == false) || (ProxyAllowed == true)) { @@ -504,12 +469,9 @@ bool is_user_allowed_to_submit_jobs( // Check limited acls if (limited_acls.is_authorized(orighost, user) == true) return(true); - else if (limited_acls.is_authorized(user_host.c_str(), user) == true) + else if (limited_acls.is_authorized(short_host.c_str(), user) == true) return(true); - if (dptr != NULL) - *dptr = '.'; - #ifdef MUNGE_AUTH sprintf(uh, "%s@%s", user.c_str(), orighost); rc = acl_check(&server.sv_attr[SRV_ATR_authusers], uh, ACL_User_Host); diff --git a/src/test/geteusernam/scaffolding.c b/src/test/geteusernam/scaffolding.c index 899a3004b1..ec849ab6df 100644 --- a/src/test/geteusernam/scaffolding.c +++ b/src/test/geteusernam/scaffolding.c @@ -12,14 +12,16 @@ #include "acl_special.hpp" #include "pbs_nodes.h" -attribute_def job_attr_def[10]; -struct server server; -char *server_host; -char *msg_orighost = strdup("Couldn't find orighost"); -int LOGLEVEL; -bool exists = true; - -acl_special limited_acls; +attribute_def job_attr_def[10]; +struct server server; +char *server_host; +char *msg_orighost = strdup("Couldn't find orighost"); +int LOGLEVEL; +bool exists = true; +std::string long_name("roshar.cosmere"); +std::string short_name("threnody"); +char *var = NULL; +acl_special limited_acls; char *site_map_user(char *uname, char *host) @@ -30,7 +32,7 @@ char *site_map_user(char *uname, char *host) char *get_variable(job *pjob, const char *variable) { - return(NULL); + return(var); } struct passwd * getpwnam_ext(char **user_buf, char * user_name) @@ -188,6 +190,17 @@ int get_svr_attr_arst( bool node_exists(const char *node_name) { + if (exists == false) + { + if (long_name == node_name) + return(true); + + if (short_name == node_name) + return(true); + } + return(exists); } +job::job() {} +job::~job() {} diff --git a/src/test/geteusernam/test_uut.c b/src/test/geteusernam/test_uut.c index d2d9a2143d..f15ea8a59d 100644 --- a/src/test/geteusernam/test_uut.c +++ b/src/test/geteusernam/test_uut.c @@ -1,16 +1,42 @@ #include "license_pbs.h" /* See here for the software license */ -#include "test_geteusernam.h" #include #include #include #include "pbs_error.h" +#include "pbs_job.h" #include "attribute.h" +#include "test_geteusernam.h" int node_exception_check(pbs_attribute *pattr, void *pobject, int actmode); -void get_user_host_from_user( std::string &user_host, const std::string user); bool is_permitted_by_node_submit(const char *orighost, int logging); +bool is_user_allowed_to_submit_jobs(job *pjob, const char *luser, char *EMsg, int logging); + +extern bool exists; +extern char *var; + + +START_TEST(test_is_user_allowed_to_submit_jobs) + { + char *user = strdup("dbeer"); + job pjob; + char emsg[1024]; + + pjob.ji_wattr[JOB_ATR_job_owner].at_val.at_str = strdup("dbeer@roshar.cosmere"); + var = NULL; + exists = false; + + // get_variable() retuns NULL, so we should fail + fail_unless(is_user_allowed_to_submit_jobs(&pjob, user, emsg, 10) == false); -extern bool exists; + // roshar.cosmere will pass but roshar won't; make sure the FQDN works + var = strdup("roshar.cosmere"); + fail_unless(is_user_allowed_to_submit_jobs(&pjob, user, emsg, 10) == true); + + // threnody will pass but threnody.cosmere won't; make sure we check the short name + var = strdup("threnody.cosmere"); + fail_unless(is_user_allowed_to_submit_jobs(&pjob, user, emsg, 10) == true); + } +END_TEST START_TEST(test_is_permitted_by_node_submit) @@ -27,22 +53,6 @@ START_TEST(test_is_permitted_by_node_submit) } END_TEST - -START_TEST(test_get_user_host_from_user) - { - std::string user_host; - std::string user; - - user = "user@hostname"; - get_user_host_from_user(user_host, user); - fail_unless(strcmp(user_host.c_str(), "hostname") == 0); - - user = "user"; - get_user_host_from_user(user_host, user); - fail_unless(user_host.size() == 0); - } -END_TEST - START_TEST(test_node_exception_check) { pbs_attribute pattr; @@ -77,12 +87,12 @@ Suite *geteusernam_suite(void) { Suite *s = suite_create("geteusernam_suite methods"); TCase *tc_core = tcase_create("test_get_user_host_from_user"); - tcase_add_test(tc_core, test_get_user_host_from_user); tcase_add_test(tc_core, test_node_exception_check); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_is_permitted_by_node_submit"); tcase_add_test(tc_core, test_is_permitted_by_node_submit); + tcase_add_test(tc_core, test_is_user_allowed_to_submit_jobs); suite_add_tcase(s, tc_core); return s; From 4c66f1f1123c689ce22bfd4de0af35306805f8b6 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 12 Apr 2017 14:39:35 -0600 Subject: [PATCH 223/312] TRQ-3993 - fix hostname lookups in torque.setup --- torque.setup.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torque.setup.in b/torque.setup.in index 65da3a0d86..15716c9b97 100644 --- a/torque.setup.in +++ b/torque.setup.in @@ -90,7 +90,7 @@ function get_ip_addrs function ip2hostname { if [ ! -z "$1" ]; then - getent hosts $1 | awk '{print $2}' + getent hosts $1 | awk '$1 == "'$1'" {print $2; exit}' fi } @@ -103,7 +103,7 @@ function get_hostnames # add hostname to the list echo "$hostname" fi - done + done | sort -u } # remove localhost from hostname list if it is not the hostname of this host From dbcdf1f7dcf39c08ff81bfcba3e9bf0024423fa8 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 12 Apr 2017 15:59:11 -0600 Subject: [PATCH 224/312] TRQ-3932. Tweak this fix. Conflicts: src/server/req_deletearray.c --- src/server/pbsd_init.c | 4 +--- src/server/req_deletearray.c | 6 ++++-- src/server/svr_jobfunc.c | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/server/pbsd_init.c b/src/server/pbsd_init.c index d5aec1a227..26a0e8302f 100644 --- a/src/server/pbsd_init.c +++ b/src/server/pbsd_init.c @@ -2032,9 +2032,7 @@ int cleanup_recovered_arrays() if (job_template_exists == FALSE) { - int i; - - for (i = 0; i < pa->ai_qs.array_size; i++) + for (int i = 0; i < pa->ai_qs.array_size; i++) { if (pa->job_ids[i] != NULL) { diff --git a/src/server/req_deletearray.c b/src/server/req_deletearray.c index e8f65e14e3..d791de8b86 100644 --- a/src/server/req_deletearray.c +++ b/src/server/req_deletearray.c @@ -158,10 +158,12 @@ int req_deletearray( pa = get_array(preq->rq_ind.rq_delete.rq_objname); - // Do not attempt to delete the array while it is still cloning + // Do not attempt to delete the array while it is still cloning. If none have been cloned, + // it's because the array was never routed. while ((pa != NULL) && ((pa->ai_qs.num_cloned < pa->ai_qs.idle_slot_limit) && - (pa->ai_qs.num_cloned != pa->ai_qs.num_jobs))) + (pa->ai_qs.num_cloned != pa->ai_qs.num_jobs)) && + (pa->ai_qs.num_cloned > 0)) { unlock_ai_mutex(pa, __func__, NULL, 10); sleep(1); diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index bbf445fb39..a4d3493de6 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -483,8 +483,10 @@ int svr_enquejob( } } - - if ((pque->qu_attr[QA_ATR_MaxUserJobs].at_flags & ATR_VFLAG_SET)) + // Do not perform this check for array subjobs; they've already been counted. + if ((pque->qu_attr[QA_ATR_MaxUserJobs].at_flags & ATR_VFLAG_SET) && + ((pjob->ji_arraystructid[0] == '\0') || + (pjob->ji_is_array_template))) { std::string uname(pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str); From e38033be083f4cdb4e0fcdbec3a9669c087fb52f Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 13 Apr 2017 11:57:10 -0600 Subject: [PATCH 225/312] TRQ-3932. Fix some invalid reads and writes when qdel -p'ing an array. Conflicts: src/server/req_deletearray.c --- src/server/svr_jobfunc.c | 50 ++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index a4d3493de6..cac322ffb3 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -813,15 +813,21 @@ int svr_dequejob( } /* the only reason to care about the error is if the job is gone */ - int rc2; - if ((rc2 = remove_job(pque->qu_jobs_array_sum, pjob)) == PBSE_JOBNOTFOUND) - return(rc2); - - if (rc2 == THING_NOT_FOUND && (LOGLEVEL >= 8)) + // Do not check array subjobs; they aren't in the summary + if ((pjob->ji_arraystructid[0] == '\0') || + (pjob->ji_is_array_template)) { - snprintf(log_buf,sizeof(log_buf), - "Could not remove job %s from qu_jobs_array_sum\n", jobid.c_str()); - log_ext(-1, __func__, log_buf, LOG_WARNING); + int rc2; + if ((rc2 = remove_job(pque->qu_jobs_array_sum, pjob)) == PBSE_JOBNOTFOUND) + return(rc2); + + if ((rc2 == THING_NOT_FOUND) && + (LOGLEVEL >= 8)) + { + snprintf(log_buf,sizeof(log_buf), + "Could not remove job %s from qu_jobs_array_sum\n", jobid.c_str()); + log_ext(-1, __func__, log_buf, LOG_WARNING); + } } pjob->ji_qhdr = NULL; @@ -851,9 +857,9 @@ int svr_dequejob( } /* the only error is if the job isn't present */ - if ((rc = remove_job(&alljobs, pjob)) == PBSE_NONE) + if (!pjob->ji_is_array_template) { - if (!pjob->ji_is_array_template) + if ((rc = remove_job(&alljobs, pjob)) == PBSE_NONE) { lock_sv_qs_mutex(server.sv_qs_mutex, __func__); @@ -874,18 +880,18 @@ int svr_dequejob( pthread_mutex_unlock(server.sv_jobstates_mutex); } - } - else if (rc == PBSE_JOBNOTFOUND) - { - /* calling functions know this return code means the job is gone */ - return(rc); - } - - if (rc == THING_NOT_FOUND && (LOGLEVEL >= 8)) - { - snprintf(log_buf,sizeof(log_buf), - "Could not remove job %s from alljobs\n", pjob->ji_qs.ji_jobid); - log_ext(-1, __func__, log_buf, LOG_WARNING); + else if (rc == PBSE_JOBNOTFOUND) + { + /* calling functions know this return code means the job is gone */ + return(rc); + } + if ((rc == THING_NOT_FOUND) && + (LOGLEVEL >= 8)) + { + snprintf(log_buf,sizeof(log_buf), + "Could not remove job %s from alljobs\n", pjob->ji_qs.ji_jobid); + log_ext(-1, __func__, log_buf, LOG_WARNING); + } } #ifndef NDEBUG From 9990e0695bc6b4d1ddfcaf07209cf3beebb927fd Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 13 Apr 2017 12:07:37 -0600 Subject: [PATCH 226/312] TRQ-3932. Fix a deadlock and cleanup better when purging an unrouteable array. --- src/server/req_deletearray.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/req_deletearray.c b/src/server/req_deletearray.c index d791de8b86..d6d49d21ea 100644 --- a/src/server/req_deletearray.c +++ b/src/server/req_deletearray.c @@ -233,8 +233,7 @@ int req_deletearray( log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); } - if (((num_skipped = delete_whole_array(pa, purge)) == NO_JOBS_IN_ARRAY) && - (purge == false)) + if ((num_skipped = delete_whole_array(pa, purge)) == NO_JOBS_IN_ARRAY) { pa_mutex.unlock(); array_delete(preq->rq_ind.rq_delete.rq_objname); From 88b3d3980d675d1b98651bcbc29b6c3963bb9526 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 13 Apr 2017 16:43:10 -0600 Subject: [PATCH 227/312] TRQ-3934. For consistency, display memory and total memory. All req fields are per task by default. Conflicts: src/test/complete_req/test_uut.c src/test/numa_chip/scaffolding.c --- src/include/req.hpp | 12 +-- src/lib/Libattr/attr_fn_complete_req.c | 8 +- src/lib/Libattr/req.cpp | 141 +++++++++++++------------ src/lib/Libutils/allocation.cpp | 4 +- src/lib/Libutils/machine.cpp | 2 +- src/lib/Libutils/numa_chip.cpp | 6 +- src/test/allocation/scaffolding.c | 2 +- src/test/allocation/test_uut.c | 7 -- src/test/complete_req/test_uut.c | 26 ++--- src/test/machine/scaffolding.c | 2 +- src/test/numa_chip/scaffolding.c | 13 ++- src/test/numa_chip/test_uut.c | 3 +- src/test/req/test_uut.c | 40 +++---- 13 files changed, 137 insertions(+), 129 deletions(-) diff --git a/src/include/req.hpp b/src/include/req.hpp index 368c19131e..080e66447d 100644 --- a/src/include/req.hpp +++ b/src/include/req.hpp @@ -116,9 +116,9 @@ extern const char *place_legacy2; class req { int execution_slots; - unsigned long mem; + unsigned long total_mem; unsigned long mem_per_task; - unsigned long swap; + unsigned long total_swap; unsigned long swap_per_task; unsigned long disk; int nodes; @@ -174,8 +174,8 @@ class req void get_values(std::vector &names, std::vector &values) const; void toString(std::string &str) const; int getExecutionSlots() const; - unsigned long getMemory() const; - unsigned long getSwap() const; + unsigned long get_total_memory() const; + unsigned long get_total_swap() const; int get_cores() const; int get_threads() const; int get_sockets() const; @@ -204,9 +204,9 @@ class req bool cgroup_preference_set() const; int get_task_allocation(unsigned int index, allocation &task_allocation) const; unsigned long long get_memory_for_host(const std::string &host) const; - unsigned long long get_memory_per_task(); + unsigned long long get_memory_per_task() const; unsigned long long get_swap_for_host(const std::string &host) const; - unsigned long long get_swap_per_task(); + unsigned long long get_swap_per_task() const; void get_task_stats(std::vector &task_index, std::vector &cput_used, std::vector &mem_used); int get_execution_slots() const; diff --git a/src/lib/Libattr/attr_fn_complete_req.c b/src/lib/Libattr/attr_fn_complete_req.c index 358fa5780c..c50a2ccb16 100644 --- a/src/lib/Libattr/attr_fn_complete_req.c +++ b/src/lib/Libattr/attr_fn_complete_req.c @@ -152,8 +152,8 @@ int decode_complete_req( unsigned long long swap; req &r = cr->get_req(i); - mem = r.getMemory(); - swap = r.getSwap(); + mem = r.get_total_memory(); + swap = r.get_total_swap(); if (swap != 0) { @@ -220,8 +220,8 @@ int decode_complete_req( unsigned long long swap; req &r = cr->get_req(i); - mem = r.getMemory(); - swap = r.getSwap(); + mem = r.get_total_memory(); + swap = r.get_total_swap(); if (swap != 0) { diff --git a/src/lib/Libattr/req.cpp b/src/lib/Libattr/req.cpp index 21e5fe8d5e..efa5d8fe98 100644 --- a/src/lib/Libattr/req.cpp +++ b/src/lib/Libattr/req.cpp @@ -34,12 +34,12 @@ const char *place_thread = "thread"; const char *place_legacy = "legacy"; const char *place_legacy2 = "legacy2"; -req::req() : execution_slots(1), mem(0), mem_per_task(0), swap(0), swap_per_task(0), disk(0), nodes(0), - socket(0), numa_nodes(0), cores(0), threads(0), thread_usage_policy(ALLOW_THREADS), - thread_usage_str(allow_threads), gpus(0), mics(0), maxtpn(0), gres(), os(), arch(), - node_access_policy(), features(), placement_str(), req_attr(), gpu_mode(), - placement_type(exclusive_none), task_count(1), pack(false), single_job_access(false), - per_task_cgroup(-1), index(0), hostlist(), task_allocations() +req::req() : execution_slots(1), total_mem(0), mem_per_task(0), total_swap(0), swap_per_task(0), + disk(0), nodes(0), socket(0), numa_nodes(0), cores(0), threads(0), + thread_usage_policy(ALLOW_THREADS), thread_usage_str(allow_threads), gpus(0), mics(0), + maxtpn(0), gres(), os(), arch(), node_access_policy(), features(), placement_str(), + req_attr(), gpu_mode(), placement_type(exclusive_none), task_count(1), pack(false), + single_job_access(false), per_task_cgroup(-1), index(0), hostlist(), task_allocations() { } @@ -48,8 +48,8 @@ req::req() : execution_slots(1), mem(0), mem_per_task(0), swap(0), swap_per_task req::req( - char *work_str) : execution_slots(1), mem(0), mem_per_task(0), swap(0), swap_per_task(0), disk(0), - nodes(0), socket(0), numa_nodes(0), cores(0), threads(0), + char *work_str) : execution_slots(1), total_mem(0), mem_per_task(0), total_swap(0), swap_per_task(0), + disk(0), nodes(0), socket(0), numa_nodes(0), cores(0), threads(0), thread_usage_policy(ALLOW_THREADS), thread_usage_str(allow_threads), gpus(0), mics(0), maxtpn(0), gres(), os(), arch(), node_access_policy(), features(), placement_str(), gpu_mode(), task_count(1), pack(false), single_job_access(false), @@ -109,9 +109,9 @@ req::req( req::req( const req &other) : execution_slots(other.execution_slots), - mem(other.mem), + total_mem(other.total_mem), mem_per_task(other.mem_per_task), - swap(other.swap), + total_swap(other.total_swap), swap_per_task(other.swap_per_task), disk(other.disk), nodes(other.nodes), @@ -541,7 +541,7 @@ int req::set_name_value_pair( rc = read_mem_value(value, this->mem_per_task); if (rc == PBSE_NONE) - this->mem = this->mem_per_task * this->task_count; + this->total_mem = this->mem_per_task * this->task_count; } else if (!strcmp(name, "gpus")) rc = parse_positive_integer(value, this->gpus); @@ -571,7 +571,7 @@ int req::set_name_value_pair( rc = read_mem_value(value, this->swap_per_task); if (rc == PBSE_NONE) - this->swap = this->swap_per_task * this->task_count; + this->total_swap = this->swap_per_task * this->task_count; } else return(PBSE_BAD_PARAMETER); @@ -1123,8 +1123,8 @@ int req::set_from_submission_string( req::req( - const std::string &resource_request) : execution_slots(1), mem(0), mem_per_task(0), swap(0), - swap_per_task(0), disk(0), nodes(0), socket(0), + const std::string &resource_request) : execution_slots(1), total_mem(0), mem_per_task(0), + total_swap(0), swap_per_task(0), disk(0), nodes(0), socket(0), numa_nodes(0), cores(0), threads(0), thread_usage_policy(ALLOW_THREADS), thread_usage_str(allow_threads), gpus(0), mics(0), @@ -1158,9 +1158,9 @@ req &req::operator =( return(*this); this->execution_slots = other.execution_slots; - this->mem = other.mem; + this->total_mem = other.total_mem; this->mem_per_task = other.mem_per_task; - this->swap = other.swap; + this->total_swap = other.total_swap; this->swap_per_task = other.swap_per_task; this->disk = other.disk; this->nodes = other.nodes; @@ -1226,27 +1226,27 @@ void req::toString( } } - if (this->mem != 0) + if (this->total_mem != 0) { - snprintf(buf, sizeof(buf), " mem: %lukb\n", this->mem); + snprintf(buf, sizeof(buf), " total_mem: %lukb\n", this->total_mem); str += buf; } if (this->mem_per_task != 0) { - snprintf(buf, sizeof(buf), " mem_per_task: %lukb\n", this->mem_per_task); + snprintf(buf, sizeof(buf), " mem: %lukb\n", this->mem_per_task); str += buf; } - if (this->swap != 0) + if (this->total_swap != 0) { - snprintf(buf, sizeof(buf), " swap: %lukb\n", this->swap); + snprintf(buf, sizeof(buf), " total_swap: %lukb\n", this->total_swap); str += buf; } if (this->swap_per_task != 0) { - snprintf(buf, sizeof(buf), " swap_per_task: %lukb\n", this->swap_per_task); + snprintf(buf, sizeof(buf), " swap: %lukb\n", this->swap_per_task); str += buf; } @@ -1387,6 +1387,9 @@ void req::toString( * * Gets the name and value of each part of this class that is set * + * NOTE: these values are usually passed to encode_complete_req() and often + * later consumed by decode_complete_req() which calls set_value() + * * For each value that is set: * name=:index value=mem != 0) + if (this->mem_per_task != 0) { - snprintf(buf, sizeof(buf), "memory.%d", this->index); + snprintf(buf, sizeof(buf), "total_memory.%d", this->index); names.push_back(buf); - snprintf(buf, sizeof(buf), "%lukb", this->mem); + snprintf(buf, sizeof(buf), "%lukb", this->total_mem); values.push_back(buf); if (this->mem_per_task != 0) { - snprintf(buf, sizeof(buf), "memory_per_task.%d", this->index); + snprintf(buf, sizeof(buf), "memory.%d", this->index); names.push_back(buf); snprintf(buf, sizeof(buf), "%lukb", this->mem_per_task); values.push_back(buf); } } - if (this->swap != 0) + if (this->swap_per_task != 0) { - snprintf(buf, sizeof(buf), "swap.%d", this->index); + snprintf(buf, sizeof(buf), "total_swap.%d", this->index); names.push_back(buf); - snprintf(buf, sizeof(buf), "%lukb", this->swap); + snprintf(buf, sizeof(buf), "%lukb", this->total_swap); values.push_back(buf); if (this->swap_per_task != 0) { - snprintf(buf, sizeof(buf), "swap_per_task.%d", this->index); + snprintf(buf, sizeof(buf), "swap.%d", this->index); names.push_back(buf); snprintf(buf, sizeof(buf), "%lukb", this->swap_per_task); values.push_back(buf); @@ -1620,6 +1623,7 @@ void req::get_values( } // END get_values() + void req::get_task_stats( std::vector &task_index, @@ -1708,10 +1712,10 @@ char *capture_until_newline_and_advance( * req[] * task count: * [lprocs: ] - * [mem: kb] - * [mem_per_task: kb] - * [swap: kb] - * [swap_per_task: kb] + * [total_mem: kb] + * [mem: kb] + * [total_swap: kb] + * [swap: kb] * [disk: kb] * [socket: ] * [numa chips: ] @@ -1779,43 +1783,45 @@ void req::set_from_string( move_past_whitespace(¤t); } - if (!strncmp(current, "mem", 3)) + if (!strncmp(current, "total_mem", 9)) { - // found mem - current += 5; // move past 'mem: ' - this->mem = strtoll(current, ¤t, 10); - this->mem_per_task = this->mem / this->task_count; + // found total mem + current += 11; // move past 'total_mem: ' + this->total_mem = strtoll(current, ¤t, 10); + this->mem_per_task = this->total_mem / this->task_count; current += 2; // move past 'kb' move_past_whitespace(¤t); } - if (!strncmp(current, "mem_per_task", 12)) + if (!strncmp(current, "mem", 3)) { - // found mem_per_task - current += 14; // move past 'mem_per_task: ' + // found mem + current += 5; // move past 'mem: ' this->mem_per_task = strtoll(current, ¤t, 10); + this->total_mem = this->mem_per_task * this->task_count; current += 2; // move past 'kb' move_past_whitespace(¤t); } - if (!strncmp(current, "swap", 4)) + if (!strncmp(current, "total_swap", 10)) { - // found swap - current += 6; // move past 'swap: ' - this->swap = strtoll(current, ¤t, 10); - this->swap_per_task = this->swap / this->task_count; + // found total_swap + current += 12; // move past 'total_swap: ' + this->total_swap = strtoll(current, ¤t, 10); + this->swap_per_task = this->total_swap / this->task_count; current += 2; // move past kb move_past_whitespace(¤t); } - if (!strncmp(current, "swap_per_task", 13)) + if (!strncmp(current, "swap", 4)) { // found swap - current += 15; // move past 'swap_per_task: ' - this->swap = strtoll(current, ¤t, 10); + current += 6; // move past 'swap: ' + this->swap_per_task = strtoll(current, ¤t, 10); + this->total_swap = this->swap_per_task * this->task_count; current += 2; // move past kb move_past_whitespace(¤t); @@ -2058,6 +2064,8 @@ void req::set_from_string( * set_value() * * Sets a value on this class identified by name + * NOTE: this is called by decode_complete_req() and should match what is produced by get_values(), + * since get_values() is called by encode_complete_req() * @param name - the name of the value to be set * @param value - the value * @return PBSE_NONE if the name is valid or PBSE_BAD_PARAMETER for an invalid name @@ -2096,23 +2104,23 @@ int req::set_value( else if (!strncmp(name, "memory", 6)) { if ((!is_default) || - (this->mem == 0)) + (this->mem_per_task == 0)) { - if ((rc = read_mem_value(value, this->mem)) != PBSE_NONE) + if ((rc = read_mem_value(value, this->mem_per_task)) != PBSE_NONE) return(rc); else - this->mem_per_task = this->mem / this->task_count; + this->total_mem = this->mem_per_task * this->task_count; } } else if (!strncmp(name, "swap", 4)) { if ((!is_default) || - (this->swap == 0)) + (this->swap_per_task == 0)) { - if ((rc = read_mem_value(value, this->swap)) != PBSE_NONE) + if ((rc = read_mem_value(value, this->swap_per_task)) != PBSE_NONE) return(rc); else - this->swap_per_task = this->swap / this->task_count; + this->total_swap = this->swap_per_task * this->task_count; } } else if (!strncmp(name, "disk", 4)) @@ -2225,7 +2233,7 @@ int req::set_value( /* - * set_value() + * set_task_value() * * Sets a value on this class identified by name * @param name - the name of the value to be set @@ -2276,16 +2284,16 @@ int req::getExecutionSlots() const return(this->execution_slots); } -unsigned long req::getMemory() const +unsigned long req::get_total_memory() const { - return(this->mem); + return(this->total_mem); } -unsigned long req::getSwap() const +unsigned long req::get_total_swap() const { - return(this->swap); + return(this->total_swap); } unsigned long req::getDisk() const @@ -2548,7 +2556,7 @@ int req::get_task_allocation( * */ -unsigned long long req::get_swap_per_task() +unsigned long long req::get_swap_per_task() const { return(this->swap_per_task); @@ -2562,7 +2570,7 @@ unsigned long long req::get_swap_per_task() * */ -unsigned long long req::get_memory_per_task() +unsigned long long req::get_memory_per_task() const { return(this->mem_per_task); @@ -2650,8 +2658,8 @@ void req::set_memory( unsigned long mem) { - this->mem = mem; - this->mem_per_task = this->mem / this->task_count; + this->total_mem = mem; + this->mem_per_task = this->total_mem / this->task_count; } void req::set_swap( @@ -2659,7 +2667,8 @@ void req::set_swap( unsigned long swap) { - this->swap = swap; + this->total_swap = swap; + this->swap_per_task = this->total_swap / this->task_count; } int req::get_execution_slots() const diff --git a/src/lib/Libutils/allocation.cpp b/src/lib/Libutils/allocation.cpp index 0a2a6e3b92..c8362b6498 100644 --- a/src/lib/Libutils/allocation.cpp +++ b/src/lib/Libutils/allocation.cpp @@ -159,7 +159,7 @@ allocation::allocation( { this->cpus = r.getExecutionSlots(); - this->memory = r.getMemory() / r.getTaskCount(); + this->memory = r.get_memory_per_task(); this->gpus = r.get_gpus(); this->mics = r.getMics(); @@ -602,7 +602,7 @@ bool allocation::partially_placed( { return ((this->cpus != r.getExecutionSlots()) || - (this->memory != r.getMemory()) || + (this->memory != r.get_memory_per_task()) || (this->gpus != r.get_gpus()) || (this->mics != r.getMics()) || ((r.getPlaceCores() > 0) && diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index 6e74f1491b..25f9b52aa3 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -949,7 +949,7 @@ int Machine::spread_place( else { // Make sure we are grabbing enough memory - unsigned long mem_needed = r.getMemory() / r.getTaskCount(); + unsigned long mem_needed = r.get_memory_per_task(); unsigned long mem_count = 0; int mem_quantity = 0; diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index 50e4374ec4..ce2f74a668 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -964,7 +964,7 @@ double Chip::how_many_tasks_fit( else cpu_tasks = this->availableThreads / max_cpus; - unsigned long memory = r.getMemory() / r.getTaskCount(); + unsigned long memory = r.get_memory_per_task(); // Memory isn't required for submission if (memory != 0) @@ -1369,7 +1369,7 @@ bool Chip::task_will_fit( { bool fits = false; int max_cpus = r.getExecutionSlots(); - hwloc_uint64_t mem_per_task = r.getMemory() / r.getTaskCount(); + hwloc_uint64_t mem_per_task = r.get_memory_per_task(); int gpus_per_task = r.get_gpus(); int mics_per_task = r.getMics(); bool cores_only = (r.getThreadUsageString() == use_cores); @@ -1995,7 +1995,7 @@ int Chip::place_task( allocation chip_alloc(master.jobid.c_str()); int tasks_placed = 0; int execution_slots_per_task = r.getExecutionSlots(); - hwloc_uint64_t mem_per_task = r.getMemory() / r.getTaskCount(); + hwloc_uint64_t mem_per_task = r.get_memory_per_task(); int practical_place = master.place_type; chip_alloc.place_type = master.place_type; diff --git a/src/test/allocation/scaffolding.c b/src/test/allocation/scaffolding.c index 0be97352a1..31c1e252c2 100644 --- a/src/test/allocation/scaffolding.c +++ b/src/test/allocation/scaffolding.c @@ -23,7 +23,7 @@ std::string req::getPlacementType() const req::req() {} -unsigned long req::getMemory() const +unsigned long long req::get_memory_per_task() const { return(1024); diff --git a/src/test/allocation/test_uut.c b/src/test/allocation/test_uut.c index 822bb2b991..cab32f7560 100644 --- a/src/test/allocation/test_uut.c +++ b/src/test/allocation/test_uut.c @@ -75,13 +75,6 @@ START_TEST(test_allocation_constructors) fail_unless(a4.memory == 1024, "mem = %d", a.memory); fail_unless(a4.cpus == 2); fail_unless(a4.cores_only == true); - - // Make sure that we show memory per task - tc = 2; - allocation a5(r); - fail_unless(a5.memory == 512, "mem = %d", a.memory); - fail_unless(a5.cpus == 2); - fail_unless(a5.cores_only == true); } END_TEST diff --git a/src/test/complete_req/test_uut.c b/src/test/complete_req/test_uut.c index 8a08b169e6..33f4fd7b1d 100644 --- a/src/test/complete_req/test_uut.c +++ b/src/test/complete_req/test_uut.c @@ -135,7 +135,7 @@ START_TEST(test_constructor) fail_unless(list1.req_count() == 2); const req &rm1 = list1.get_req(0); - fail_unless(rm1.getMemory() == 13653, "mem is %lu", rm1.getMemory()); + fail_unless(rm1.get_total_memory() == 13653, "mem is %lu", rm1.get_total_memory()); resources.clear(); add_resource(resources, "size", NULL, 20, -1); @@ -145,7 +145,7 @@ START_TEST(test_constructor) const req &r = list2.get_req(0); fail_unless(r.getTaskCount() == 20); fail_unless(r.getExecutionSlots() == 1); - fail_unless(r.getMemory() == 40, "mem is %lu", r.getMemory()); + fail_unless(r.get_total_memory() == 40, "mem is %lu", r.get_total_memory()); resources.clear(); add_resource(resources, "ncpus", NULL, 16, -1); @@ -155,7 +155,7 @@ START_TEST(test_constructor) const req &rl = list3.get_req(0); fail_unless(rl.getTaskCount() == 1); fail_unless(rl.getExecutionSlots() == 16); - fail_unless(rl.getMemory() == 40, "mem is %lu", rl.getMemory()); + fail_unless(rl.get_total_memory() == 40, "mem is %lu", rl.get_total_memory()); resources.clear(); add_resource(resources, "nodes", "1:ppn=2", 16, -1); @@ -165,7 +165,7 @@ START_TEST(test_constructor) const req &rl2 = list4.get_req(0); fail_unless(rl2.getTaskCount() == 1); // pmem should get multiplied because nodes=1:ppn=2 - fail_unless(rl2.getMemory() == 160, "pmem is %lu", rl2.getMemory()); + fail_unless(rl2.get_total_memory() == 160, "pmem is %lu", rl2.get_total_memory()); resources.clear(); add_resource(resources, "nodes", "1:ppn=2", 16, -1); @@ -174,9 +174,9 @@ START_TEST(test_constructor) fail_unless(list5.req_count() == 1); const req &rl3 = list5.get_req(0); fail_unless(rl3.getTaskCount() == 1); - fail_unless(rl3.getMemory() == 0, "pvmem is %lu", rl3.getMemory()); + fail_unless(rl3.get_total_memory() == 0, "pvmem is %lu", rl3.get_total_memory()); // pvmem should get multiplied because nodes=1:ppn=2 is one task - fail_unless(rl3.getSwap() == 160, "pvmem is %lu", rl3.getSwap()); + fail_unless(rl3.get_total_swap() == 160, "pvmem is %lu", rl3.get_total_swap()); resources.clear(); add_resource(resources, "nodes", "1:ppn=2", 16, -1); @@ -185,8 +185,8 @@ START_TEST(test_constructor) fail_unless(list6.req_count() == 1); const req &rl4 = list6.get_req(0); fail_unless(rl4.getTaskCount() == 1); - fail_unless(rl4.getMemory() == 0, "vmem is %lu", rl4.getMemory()); - fail_unless(rl4.getSwap() == 80, "vmem is %lu", rl4.getSwap()); + fail_unless(rl4.get_total_memory() == 0, "vmem is %lu", rl4.get_total_memory()); + fail_unless(rl4.get_total_swap() == 80, "vmem is %lu", rl4.get_total_swap()); resources.clear(); add_resource(resources, "procs", NULL, 2, -1); @@ -196,7 +196,7 @@ START_TEST(test_constructor) const req &rl7 = list7.get_req(0); fail_unless(rl7.getTaskCount() == 2, "task count is %d", rl7.getTaskCount()); // 1024 * 2 = 2048, multiply by 2 because procs=2 - fail_unless(rl7.getMemory() == 2048, "mem = %lu", rl7.getMemory()); + fail_unless(rl7.get_total_memory() == 2048, "mem = %lu", rl7.get_total_memory()); // Make sure that we'll set memory to the higher of pmem and mem, and set swap // as well for the same job @@ -209,8 +209,8 @@ START_TEST(test_constructor) fail_unless(list8.req_count() == 1); const req &rl8 = list8.get_req(0); fail_unless(rl8.getTaskCount() == 1); - fail_unless(rl8.getMemory() == 4096); - fail_unless(rl8.getSwap() == 8192); + fail_unless(rl8.get_total_memory() == 4096); + fail_unless(rl8.get_total_swap() == 8192); resources.clear(); add_resource(resources, "procs", NULL, 1, -1); @@ -219,7 +219,7 @@ START_TEST(test_constructor) complete_req list9(&resources, 1, false); fail_unless(list9.req_count() == 1); const req &rl9 = list9.get_req(0); - fail_unless(rl9.getMemory() == 5000); + fail_unless(rl9.get_total_memory() == 5000); } END_TEST @@ -233,7 +233,7 @@ START_TEST(test_constructor_oldstyle_req) fail_unless(list1.req_count() == 1); const req &rm1 = list1.get_req(0); - fail_unless(rm1.getMemory() == 0, "mem is %lu", rm1.getMemory()); + fail_unless(rm1.get_total_memory() == 0, "mem is %lu", rm1.get_total_memory()); } END_TEST diff --git a/src/test/machine/scaffolding.c b/src/test/machine/scaffolding.c index bb50329613..f1db6d6fb8 100644 --- a/src/test/machine/scaffolding.c +++ b/src/test/machine/scaffolding.c @@ -254,7 +254,7 @@ int Socket::get_total_gpus() const return(0); } -unsigned long req::getMemory() const +unsigned long long req::get_memory_per_task() const { return(req_mem); } diff --git a/src/test/numa_chip/scaffolding.c b/src/test/numa_chip/scaffolding.c index d460aad0c6..8c6b02d89d 100644 --- a/src/test/numa_chip/scaffolding.c +++ b/src/test/numa_chip/scaffolding.c @@ -137,11 +137,11 @@ int get_machine_total_memory(hwloc_topology_t topology, unsigned long *memory) return(PBSE_NONE); } -req::req() : mem(0), cores(0), threads(0), gpus(0), mics(0), task_count(1) {} +req::req() : mem_per_task(0), cores(0), threads(0), gpus(0), mics(0), task_count(1) {} -unsigned long req::getMemory() const +unsigned long long req::get_memory_per_task() const { - return(this->mem); + return(this->mem_per_task); } int req::getTaskCount() const @@ -180,7 +180,12 @@ int req::set_value(const char *name, const char *value, bool is_default) if (!strcmp(name, "lprocs")) this->execution_slots = atoi(value); else if (!strcmp(name, "memory")) - this->mem = strtol(value, NULL, 10); + this->mem_per_task = strtol(value, NULL, 10); + else if (!strcmp(name, "total_memory")) + { + this->total_mem = strtol(value, NULL, 10); + this->mem_per_task = this->total_mem / this->task_count; + } else if (!strcmp(name, GPUS)) this->gpus = atoi(value); else if (!strcmp(name, MICS)) diff --git a/src/test/numa_chip/test_uut.c b/src/test/numa_chip/test_uut.c index 65a765750c..41c38db5b8 100644 --- a/src/test/numa_chip/test_uut.c +++ b/src/test/numa_chip/test_uut.c @@ -745,11 +745,12 @@ START_TEST(test_how_many_tasks_fit) // Make sure we handle memory per task req rpt; rpt.set_value("lprocs", "1", false); - rpt.set_value("memory", "5kb", false); + rpt.set_value("total_memory", "5kb", false); tasks = c.how_many_tasks_fit(rpt, 0); fail_unless(tasks == 1, "%d tasks fit, expected 1", tasks); rpt.set_value("task_count", "5", false); + rpt.set_value("total_memory", "5kb", false); tasks = c.how_many_tasks_fit(rpt, 0); fail_unless(tasks == 5, "%d tasks fit, expected 5", tasks); diff --git a/src/test/req/test_uut.c b/src/test/req/test_uut.c index 869b73774e..f891eb0f5a 100644 --- a/src/test/req/test_uut.c +++ b/src/test/req/test_uut.c @@ -172,10 +172,10 @@ START_TEST(test_get_set_values) fail_unless(names[0] == "task_count.0"); fail_unless(names[1] == "lprocs.0"); - fail_unless(names[2] == "memory.0"); - fail_unless(names[3] == "memory_per_task.0"); - fail_unless(names[4] == "swap.0"); - fail_unless(names[5] == "swap_per_task.0"); + fail_unless(names[2] == "total_memory.0"); + fail_unless(names[3] == "memory.0"); + fail_unless(names[4] == "total_swap.0"); + fail_unless(names[5] == "swap.0"); fail_unless(names[6] == "disk.0"); fail_unless(names[7] == "socket.0"); fail_unless(names[8] == "gpus.0"); @@ -192,10 +192,10 @@ START_TEST(test_get_set_values) fail_unless(values[0] == "5"); fail_unless(values[1] == "all"); - fail_unless(values[2] == "1048576kb", "value: %s", values[2].c_str()); - fail_unless(values[3] == "209715kb", "value: %s", values[3].c_str()); - fail_unless(values[4] == "1024kb"); - fail_unless(values[5] == "204kb"); + fail_unless(values[2] == "5242880kb", "value: %s", values[2].c_str()); + fail_unless(values[3] == "1048576kb", "value: %s", values[3].c_str()); + fail_unless(values[4] == "5120kb", "value: %s", values[4].c_str()); + fail_unless(values[5] == "1024kb"); fail_unless(values[6] == "10000000kb"); fail_unless(values[7] == "1"); fail_unless(values[8] == "2"); @@ -358,8 +358,8 @@ START_TEST(test_constructors) fail_unless(r.getOS().size() == 0); fail_unless(r.getGres().size() == 0); fail_unless(r.getDisk() == 0); - fail_unless(r.getSwap() == 0); - fail_unless(r.getMemory() == 0); + fail_unless(r.get_total_swap() == 0); + fail_unless(r.get_total_memory() == 0); fail_unless(r.getExecutionSlots() == 1, "slots: %d", r.getExecutionSlots()); std::string req2("5:lprocs=4:memory=12gb:place=socket=2:usecores:pack:gpus=2:mics=1:gres=matlab=1:feature=fast"); @@ -370,8 +370,8 @@ START_TEST(test_constructors) fail_unless(r2.getOS().size() == 0); fail_unless(r2.getGres() == "matlab=1"); fail_unless(r2.getDisk() == 0); - fail_unless(r2.getSwap() == 0); - fail_unless(r2.getMemory() == 12 * 1024 * 1024 * 5); + fail_unless(r2.get_total_swap() == 0); + fail_unless(r2.get_total_memory() == 12 * 1024 * 1024 * 5); fail_unless(r2.get_memory_per_task() == 12 * 1024 * 1024); fail_unless(r2.getExecutionSlots() == 4); fail_unless(r2.getFeatures() == "fast", "features '%s'", r2.getFeatures().c_str()); @@ -383,8 +383,8 @@ START_TEST(test_constructors) fail_unless(copy_r2.getOS().size() == 0); fail_unless(copy_r2.getGres() == "matlab=1"); fail_unless(copy_r2.getDisk() == 0); - fail_unless(copy_r2.getSwap() == 0); - fail_unless(copy_r2.getMemory() == 12 * 1024 * 1024 * 5); + fail_unless(copy_r2.get_total_swap() == 0); + fail_unless(copy_r2.get_total_memory() == 12 * 1024 * 1024 * 5); fail_unless(copy_r2.get_memory_per_task() == 12 * 1024 * 1024); fail_unless(copy_r2.getExecutionSlots() == 4); fail_unless(copy_r2.getFeatures() == "fast", "features '%s'", copy_r2.getFeatures().c_str()); @@ -469,7 +469,7 @@ START_TEST(test_equals_operator) r2 = r; fail_unless(r2.getExecutionSlots() == ALL_EXECUTION_SLOTS); - fail_unless(r2.getMemory() == 1024 * 1024 * 1024); + fail_unless(r2.get_total_memory() == 1024 * 1024 * 1024); fail_unless(r2.getMaxtpn() == 4); fail_unless(r2.get_gpu_mode() == "exclusive_thread"); fail_unless(r2.getReqAttr() == "matlab>=7", "reqattr: '%s'", r2.getReqAttr().c_str()); @@ -525,9 +525,9 @@ START_TEST(test_set_from_string) fail_unless(r.getThreadUsageString() == "usecores", r.getThreadUsageString().c_str()); fail_unless(r.getFeatures() == "fast"); fail_unless(r.getExecutionSlots() == ALL_EXECUTION_SLOTS); - fail_unless(r.getMemory() == 10000); - fail_unless(r.get_memory_per_task() == 10000 / 10, "%d per task", r.get_memory_per_task()); - fail_unless(r.getSwap() == 1024); + fail_unless(r.get_total_memory() == 100000); + fail_unless(r.get_memory_per_task() == 10000, "%d per task", r.get_memory_per_task()); + fail_unless(r.get_total_swap() == 10240); fail_unless(r.getDisk() == 10000000); fail_unless(r.getTaskCount() == 10); std::vector l; @@ -605,10 +605,10 @@ START_TEST(test_get_memory_for_host) r2.set_value("hostlist", "napali:ppn=32", false); mem = r.get_memory_for_host(host); - fail_unless(mem == 204); // 1024kb / 5 tasks = 204kb for this node + fail_unless(mem == 1024); mem = r2.get_memory_for_host(host); - fail_unless(mem == 1024); // 1024kb / 1 task = 1024kb for this node + fail_unless(mem == 1024); host = "right said fred"; mem = r.get_memory_for_host(host); From 19d627efcc1d39c66038cae63850496edd062fd8 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 14 Apr 2017 09:49:03 -0600 Subject: [PATCH 228/312] Fix potential memory leak. --- src/include/container.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/include/container.hpp b/src/include/container.hpp index 681ae41bb9..8fa9e9d0b1 100644 --- a/src/include/container.hpp +++ b/src/include/container.hpp @@ -256,7 +256,11 @@ class item_container item *pItem = new item(id,it); if (insert_thing(pItem) < 0) + { + delete pItem; return false; + } + return true; } @@ -280,7 +284,10 @@ class item_container item *pItem = new item(id,it); if (insert_thing_after(pItem,index) < 0) + { + delete pItem; return false; + } return true; } @@ -310,7 +317,10 @@ class item_container item *pItem = new item(id,it); if (insert_thing_before(pItem,iter) < 0) + { + delete pItem; return false; + } return true; } @@ -349,7 +359,10 @@ class item_container item *pItem = new item(id,it); if (insert_thing_before(pItem,index) != PBSE_NONE) + { + delete pItem; return false; + } return true; } From 14abbd6ae62682983bb2b5db0e49d6eb01c7e398 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 14 Apr 2017 15:50:30 -0600 Subject: [PATCH 229/312] TRQ-3997. Make it easier to upgrade from Torque 6.0.* and 6.1.0 --- src/include/machine.hpp | 9 ++ src/lib/Libutils/machine.cpp | 116 ++++++++++++--- src/lib/Libutils/numa_chip.cpp | 236 ++++++++++++++++++++++++++++++- src/lib/Libutils/numa_socket.cpp | 43 ++++++ 4 files changed, 382 insertions(+), 22 deletions(-) diff --git a/src/include/machine.hpp b/src/include/machine.hpp index 3523c813de..ddb2be8592 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -142,6 +142,7 @@ class Chip Chip(); Chip(int execution_slots, int &es_remainder, int &per_numa_remainder); Chip(const Json::Value &layout, std::vector &valid_ids); + Chip(const std::string &legacy_layout, std::vector &valid_ids); Chip(const Chip &other); Chip &operator=(const Chip &other); ~Chip(); @@ -168,6 +169,9 @@ class Chip void parse_values_from_json_string(const Json::Value &layout, std::string &cores, std::string &threads, std::string &gpus, std::string &mics, std::vector &valid_ids); + void legacy_parse_values_from_json_string(const std::string &legacy_layout, std::string &cores, + std::string &threads, std::string &gpus, std::string &mics, + std::vector &valid_ids); #ifdef NVIDIA_GPUS #ifdef NVML_API @@ -209,6 +213,8 @@ class Chip int reserve_accelerator(int type); void free_accelerators(allocation &a); void free_accelerator(int index, int type); + void legacy_initialize_allocations(char *layout, std::vector &valid_ids); + void legacy_initialize_allocation(char *layout, std::vector &valid_ids); void initialize_allocations(const Json::Value &layout, std::vector &valid_ids); void initialize_allocation(const Json::Value &layout, std::vector &valid_ids); void aggregate_allocation(allocation &a); @@ -249,6 +255,7 @@ class Socket Socket(); Socket(int execution_slots, int numa_nodes, int &es_remainder); Socket(const Json::Value &layout, std::vector &valid_ids); + Socket(const std::string &legacy_layout, std::vector &valid_ids); ~Socket(); Socket &operator=(const Socket &other); int initializeSocket(hwloc_obj_t obj); @@ -318,6 +325,7 @@ class Machine #endif void initialize_from_json(const string &json_str, vector &valid_ids); + int legacy_initialize_from_json(const string &json_str, vector &valid_ids); public: Machine& operator=(const Machine& newMachine); @@ -368,6 +376,7 @@ class Machine bool is_initialized() const; void reinitialize_from_json(const std::string &json_layout, std::vector &valid_ids); void compare_remaining_values(allocation &remaining, const char *caller) const; + void clear(); }; extern Machine this_node; diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index 25f9b52aa3..75539fc566 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -156,6 +156,75 @@ void Machine::update_internal_counts() +/* + * legacy_initialize_from_json() + * + * Exists to help people transition from 6.1.0. Deprecated. + */ + +int Machine::legacy_initialize_from_json( + + const std::string &json_str, + std::vector &valid_ids) + + { + int rc = PBSE_NONE; + + const char *socket_str = "\"socket\":{"; + std::size_t socket_begin = json_str.find(socket_str); + + while (socket_begin != std::string::npos) + { + std::size_t next = json_str.find(socket_str, socket_begin + 1); + std::string one_socket = json_str.substr(socket_begin, next - socket_begin); + + Socket s(one_socket, valid_ids); + + this->sockets.push_back(s); + this->totalSockets++; + + socket_begin = next; + } + + if (this->totalSockets == 0) + rc = -1; + else + { + update_internal_counts(); + this->initialized = true; + } + + return(rc); + } // END legacy_initialize_from_json() + + + +/* + * Initialize everything in machine to 0 + */ + +void Machine::clear() + + { + memset(allowed_cpuset_string, 0, MAX_CPUSET_SIZE); + memset(allowed_nodeset_string, 0, MAX_NODESET_SIZE); + this->hardwareStyle = 0; + this->totalMemory = 0; + this->totalSockets = 0; + this->totalChips = 0; + this->totalCores = 0; + this->totalThreads = 0; + this->availableSockets = 0; + this->availableChips = 0; + this->availableCores = 0; + this->availableThreads = 0; + this->sockets.clear(); + this->NVIDIA_device.clear(); + this->allocations.clear(); + } + + + void Machine::initialize_from_json( const std::string &json_str, @@ -177,16 +246,37 @@ void Machine::initialize_from_json( this->sockets.push_back(s); this->totalSockets++; } - + update_internal_counts(); - this->initialized = true; + + if (this->totalCores == 0) + { + this->clear(); + + if (this->legacy_initialize_from_json(json_str, valid_ids) != PBSE_NONE) + { + char log_buf[LOCAL_LOG_BUF_SIZE]; + + snprintf(log_buf, sizeof(log_buf), "Couldn't initialize from json: '%s'", json_str.c_str()); + log_err(-1, __func__, log_buf); + } + } + else + { + this->initialized = true; + } } catch (...) { - char log_buf[LOCAL_LOG_BUF_SIZE]; + this->clear(); - snprintf(log_buf, sizeof(log_buf), "Couldn't initialize from json: '%s'", json_str.c_str()); - log_err(-1, __func__, log_buf); + if (this->legacy_initialize_from_json(json_str, valid_ids) != PBSE_NONE) + { + char log_buf[LOCAL_LOG_BUF_SIZE]; + + snprintf(log_buf, sizeof(log_buf), "Couldn't initialize from json: '%s'", json_str.c_str()); + log_err(-1, __func__, log_buf); + } } } // END initialize_from_json() @@ -198,21 +288,7 @@ void Machine::reinitialize_from_json( std::vector &valid_ids) { - memset(allowed_cpuset_string, 0, MAX_CPUSET_SIZE); - memset(allowed_nodeset_string, 0, MAX_NODESET_SIZE); - this->hardwareStyle = 0; - this->totalMemory = 0; - this->totalSockets = 0; - this->totalChips = 0; - this->totalCores = 0; - this->totalThreads = 0; - this->availableSockets = 0; - this->availableChips = 0; - this->availableCores = 0; - this->availableThreads = 0; - this->sockets.clear(); - this->NVIDIA_device.clear(); - this->allocations.clear(); + this->clear(); this->initialize_from_json(json_layout, valid_ids); } // END reinitialize_from_json() diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index ce2f74a668..ede678836b 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -122,6 +122,70 @@ Chip::Chip( +/* + * legacy_parse_values_from_json_string() + * + * Deprecated. This exists to help people upgrade. + */ + +void Chip::legacy_parse_values_from_json_string( + + const std::string &json_layout, + std::string &cores, + std::string &threads, + std::string &gpus, + std::string &mics, + std::vector &valid_ids) + + { + char *work_str = strdup(json_layout.c_str()); + char *ptr = strstr(work_str, "os_index\":"); + char *val = work_str; + + if (ptr != NULL) + { + val = ptr + strlen("os_index\":"); + this->id = strtol(val, &val, 10); + } + + if ((ptr = strstr(val, "cores\":")) != NULL) + { + val = ptr + strlen("cores\":") + 1; // add 1 for the open quote + capture_until_close_character(&val, cores, '"'); + } + + if ((ptr = strstr(val, "threads\":")) != NULL) + { + val = ptr + strlen("threads\":") + 1; // add 1 for the open quote + capture_until_close_character(&val, threads, '"'); + } + + if ((ptr = strstr(val, "mem\":")) != NULL) + { + val = ptr + strlen("mem\":"); + this->memory = strtol(val, &val, 10); + this->available_memory = this->memory; + } + + if ((ptr = strstr(val, "gpus\":")) != NULL) + { + val = ptr + strlen("gpus\":") + 1; + capture_until_close_character(&val, gpus, '"'); + } + + if ((ptr = strstr(val, "mics\":")) != NULL) + { + val = ptr + strlen("mics\":") + 1; + capture_until_close_character(&val, mics, '"'); + } + + initialize_allocations(val, valid_ids); + + free(work_str); + } // END legacy_parse_values_from_json_string() + + + /* * parse_values_from_json_string() */ @@ -196,6 +260,139 @@ void Chip::initialize_cores_from_strings( +/* + * legacy_initialize_allocation() + * + * This function is only to support people upgrading from 6.1.0. Deprecated. We should + * delete it as soon as it's reasonable to do so. + */ + +void Chip::legacy_initialize_allocation( + + char *allocation_str, + std::vector &valid_ids) + + { + allocation a; + char *ptr = strstr(allocation_str, "jobid\":"); + char *val = allocation_str; + std::string tmp_val; + + if (ptr != NULL) + { + val = ptr + 8; // move past "jobid\":\"" + capture_until_close_character(&val, tmp_val, '"'); + a.jobid = tmp_val; + } + + // check if the id is valid + bool id_valid = false; + for (size_t i = 0; i < valid_ids.size(); i++) + { + if (valid_ids[i] == a.jobid) + { + id_valid = true; + break; + } + } + + if (id_valid == true) + { + ptr = strstr(val, "cpus\":"); + if (ptr != NULL) + { + val = ptr + 7; // move past "cpus\":\"" + capture_until_close_character(&val, tmp_val, '"'); + translate_range_string_to_vector(tmp_val.c_str(), a.cpu_indices); + } + + ptr = strstr(val, "mem\":"); + if (ptr != NULL) + { + val = ptr + 5; // move past "mem\":" + a.memory = strtol(val, &val, 10); + } + + ptr = strstr(val, "exclusive\":"); + if (ptr != NULL) + { + val = ptr + 11; // move past "exclusive\":" + a.place_type = strtol(val, &val, 10); + } + + ptr = strstr(val, "cores_only\":"); + if (ptr != NULL) + { + val = ptr + 12; // move past "cores_only\":" + a.cores_only = (bool)strtol(val, &val, 10); + } + + ptr = strstr(val, "gpus\":"); + if (ptr != NULL) + { + val = ptr + 7; // move past "gpus\":\" + capture_until_close_character(&val, tmp_val, '"'); + translate_range_string_to_vector(tmp_val.c_str(), a.gpu_indices); + } + + ptr = strstr(val, "mics\":"); + if (ptr != NULL) + { + val = ptr + 7; // move past "mics\":\" + capture_until_close_character(&val, tmp_val, '"'); + translate_range_string_to_vector(tmp_val.c_str(), a.mic_indices); + } + + a.mem_indices.push_back(this->id); + this->allocations.push_back(a); + } + + } // END legacy_initialize_allocation() + + + +/* + * legacy_initialize_allocations() + * + * This function is only to support people upgrading from 6.1.0. Deprecated. We should + * delete it as soon as it's reasonable to do so. + */ + +void Chip::legacy_initialize_allocations( + + char *allocations, + std::vector &valid_ids) + + { + static const char *allocation_start = "allocation\":{"; + static const int allocation_start_len = strlen(allocation_start); + + if ((allocations == NULL) || + (*allocations == '\0')) + return; + + char *current = strstr(allocations, allocation_start); + char *next; + + while (current != NULL) + { + current += allocation_start_len; + next = strstr(current, allocation_start); + if (next != NULL) + { + // Make sure there's a termination to the current string + *next = '\0'; + } + + initialize_allocation(current, valid_ids); + + current = next; + } + + } // END legacy_initialize_allocations() + + + /* * initialize_allocation() * @@ -288,7 +485,7 @@ void Chip::initialize_allocations( /* - * reserce_allocation_resources() + * reserve_allocation_resources() * * @param a - the allocation that needs to be */ @@ -419,6 +616,42 @@ void Chip::initialize_accelerators_from_strings( +/* + * This is a legacy constructor for those transitioning from 6.1.0. Delete ASAP + */ + +Chip::Chip( + + const std::string &json_layout, + std::vector &valid_ids) : id(0), totalCores(0), totalThreads(0), availableCores(0), + availableThreads(0), total_gpus(0), available_gpus(0), + total_mics(0), available_mics(0), chip_exclusive(false), + memory(0), available_memory(0), cores(), devices(), + allocations() + + { + memset(chip_cpuset_string, 0, MAX_CPUSET_SIZE); + memset(chip_nodeset_string, 0, MAX_NODESET_SIZE); + + if (json_layout.size() == 0) + return; + + std::string cores; + std::string threads; + std::string gpus; + std::string mics; + + legacy_parse_values_from_json_string(json_layout, cores, threads, gpus, mics, valid_ids); + + initialize_cores_from_strings(cores, threads); + + initialize_accelerators_from_strings(gpus, mics); + + adjust_open_resources(); + } + + + /* * Creates a numa chip from this json * @@ -462,7 +695,6 @@ Chip::Chip( std::string gpus; std::string mics; - parse_values_from_json_string(layout, cores, threads, gpus, mics, valid_ids); initialize_cores_from_strings(cores, threads); diff --git a/src/lib/Libutils/numa_socket.cpp b/src/lib/Libutils/numa_socket.cpp index f718e2397a..4faf4ad8a2 100644 --- a/src/lib/Libutils/numa_socket.cpp +++ b/src/lib/Libutils/numa_socket.cpp @@ -46,6 +46,49 @@ Socket::Socket( +/* + * Legacy constructor for those upgrading from 6.1.0 + */ + +Socket::Socket( + + const std::string &json_layout, + std::vector &valid_ids) : id(0), memory(0), totalCores(0), totalThreads(0), + availableCores(0), availableThreads(0), chips(), + socket_exclusive(false) + + { + const char *chip_str = "\"numanode\":{"; + const char *os_str = "\"os_index\":"; + std::size_t chip_begin = json_layout.find(chip_str); + std::size_t os_begin = json_layout.find(os_str); + + memset(socket_cpuset_string, 0, MAX_CPUSET_SIZE); + memset(socket_nodeset_string, 0, MAX_NODESET_SIZE); + + if ((os_begin == std::string::npos) || + (os_begin > chip_begin)) + return; + else + { + std::string os = json_layout.substr(os_begin + strlen(os_str)); + this->id = strtol(os.c_str(), NULL, 10); + } + + while (chip_begin != std::string::npos) + { + std::size_t next = json_layout.find(chip_str, chip_begin + 1); + std::string one_chip = json_layout.substr(chip_begin, next - chip_begin); + + Chip c(one_chip, valid_ids); + this->chips.push_back(c); + + chip_begin = next; + } + } + + + /* * Builds a copy of the machine's layout in from json which has no whitespace but * if it did it'd look like: From b72474f4c34eeee65f17e756347f4645a5edd2ca Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 14 Apr 2017 16:18:17 -0600 Subject: [PATCH 230/312] TRQ-3934. Fix a silly mistake I made. --- src/lib/Libattr/req.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/lib/Libattr/req.cpp b/src/lib/Libattr/req.cpp index efa5d8fe98..4240265d17 100644 --- a/src/lib/Libattr/req.cpp +++ b/src/lib/Libattr/req.cpp @@ -2101,6 +2101,15 @@ int req::set_value( } } } + else if (!strncmp(name, "total_memory", 12)) + { + if ((!is_default) || + (this->mem_per_task == 0)) + { + if ((rc = read_mem_value(value, this->total_mem)) != PBSE_NONE) + return(rc); + } + } else if (!strncmp(name, "memory", 6)) { if ((!is_default) || @@ -2112,6 +2121,15 @@ int req::set_value( this->total_mem = this->mem_per_task * this->task_count; } } + else if (!strncmp(name, "total_swap", 10)) + { + if ((!is_default) || + (this->mem_per_task == 0)) + { + if ((rc = read_mem_value(value, this->total_swap)) != PBSE_NONE) + return(rc); + } + } else if (!strncmp(name, "swap", 4)) { if ((!is_default) || From 16de8dcf50468df5dfb4d71cbeef4909ace1e107 Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 17 Apr 2017 13:45:04 -0600 Subject: [PATCH 231/312] TRQ-3997. Add some scaffolding definitions. --- src/test/machine/scaffolding.c | 5 +++++ src/test/numa_socket/scaffolding.c | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/src/test/machine/scaffolding.c b/src/test/machine/scaffolding.c index f1db6d6fb8..76cf63448a 100644 --- a/src/test/machine/scaffolding.c +++ b/src/test/machine/scaffolding.c @@ -93,6 +93,11 @@ Socket::Socket(const Json::Value &json_layout, std::vector &valid_i json_socket++; } +Socket::Socket(const std::string &json_layout, std::vector &valid_ids) + { + json_socket++; + } + Socket::Socket() { } diff --git a/src/test/numa_socket/scaffolding.c b/src/test/numa_socket/scaffolding.c index 3cf030ab55..bafc901bea 100644 --- a/src/test/numa_socket/scaffolding.c +++ b/src/test/numa_socket/scaffolding.c @@ -109,6 +109,10 @@ Chip::Chip(const Json::Value &json_layout, std::vector &valid_ids) json_chip++; } +Chip::Chip(const std::string &json_layout, std::vector &valid_ids) + { + } + Chip::Chip(const Chip &other) { } From 68af5698429fa454cae34f45ba432a7a080e77ef Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 18 Apr 2017 17:08:06 -0600 Subject: [PATCH 232/312] TRQ-3934. Undo a piece of this change. --- src/lib/Libattr/complete_req.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Libattr/complete_req.cpp b/src/lib/Libattr/complete_req.cpp index 540daa94cd..ca8db949d7 100644 --- a/src/lib/Libattr/complete_req.cpp +++ b/src/lib/Libattr/complete_req.cpp @@ -168,7 +168,8 @@ complete_req::complete_req( } unsigned long long vmem_per_task = mem_values[_VMEM_]; - vmem_per_task /= task_count; + if (legacy_vmem == false) + vmem_per_task /= task_count; if (this->reqs.size() > 0) { From 6e8a3c286b0ca98ece0fc33bebc4965da7455dde Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 19 Apr 2017 23:18:53 +0000 Subject: [PATCH 233/312] TRQ-3994 - fix server deadlock --- src/lib/Libutils/u_misc.c | 38 +++++++++++++++++++++++++++++++++----- src/server/job_array.cpp | 4 +++- src/test/u_misc/test_uut.c | 9 +++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/lib/Libutils/u_misc.c b/src/lib/Libutils/u_misc.c index 6653f5e4b3..ca24ed1320 100644 --- a/src/lib/Libutils/u_misc.c +++ b/src/lib/Libutils/u_misc.c @@ -84,6 +84,7 @@ #include #include #include +#include #include "utils.h" #include "resource.h" @@ -336,11 +337,14 @@ int translate_range_string_to_vector( std::vector &indices) { - char *str = strdup(range_string); - char *ptr = str; - int prev = 0; - int curr; - int rc = PBSE_NONE; + char *str = strdup(range_string); + char *ptr = str; + int prev = 0; + int curr; + int rc = PBSE_NONE; + // use set to hold values added to vector + // since less expensive to check for duplicate entries + std::set vector_values; while (is_whitespace(*ptr)) ptr++; @@ -370,19 +374,43 @@ int translate_range_string_to_vector( curr = strtol(ptr, &ptr, 10); + if (prev >= curr) + { + // invalid range + rc = -1; + break; + } + while (prev <= curr) { + if (vector_values.insert(prev).second == false) + { + // duplicate entry + rc = -1; + break; + } + indices.push_back(prev); prev++; } + if (rc != PBSE_NONE) + break; + while ((*ptr == ',') || (is_whitespace(*ptr))) ptr++; } else { + if (vector_values.insert(prev).second == false) + { + // duplicate entry + rc = -1; + break; + } + indices.push_back(prev); while ((*ptr == ',') || diff --git a/src/server/job_array.cpp b/src/server/job_array.cpp index f5261a7888..47d4da9b4e 100644 --- a/src/server/job_array.cpp +++ b/src/server/job_array.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "array.h" #include "list_link.h" @@ -259,7 +260,8 @@ int job_array::parse_array_request( this->ai_qs.num_jobs = this->uncreated_ids.size(); // size of array is the biggest index + 1 - this->ai_qs.array_size = this->uncreated_ids[this->uncreated_ids.size() - 1] + 1; + this->ai_qs.array_size = *std::max_element(this->uncreated_ids.begin(), + this->uncreated_ids.end()) + 1; if (get_svr_attr_l(SRV_ATR_MaxArraySize, &max_array_size) == PBSE_NONE) { diff --git a/src/test/u_misc/test_uut.c b/src/test/u_misc/test_uut.c index 4366a76fde..5efcb1336c 100644 --- a/src/test/u_misc/test_uut.c +++ b/src/test/u_misc/test_uut.c @@ -116,6 +116,15 @@ START_TEST(test_translate_range_string_to_vector) indices.clear(); fail_unless(translate_range_string_to_vector("0--1", indices) != PBSE_NONE); + + indices.clear(); + fail_unless(translate_range_string_to_vector("3-1", indices) != PBSE_NONE); + + indices.clear(); + fail_unless(translate_range_string_to_vector("10,10", indices) != PBSE_NONE); + + indices.clear(); + fail_unless(translate_range_string_to_vector("5-5", indices) != PBSE_NONE); } END_TEST From 1c985a013f950cc3bf648875db47d15d763ee12c Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 20 Apr 2017 11:40:21 -0600 Subject: [PATCH 234/312] Fix a regression around submitting a large array and then immediately shutting down. --- src/server/issue_request.c | 6 ++++-- src/server/job_func.c | 21 +++++++++++++++------ src/server/job_recov.c | 7 ++++--- src/server/node_manager.c | 13 +++++-------- src/server/reply_send.c | 5 +++-- src/server/req_deletearray.c | 14 +++++++++++--- src/server/req_jobobit.c | 5 +++-- 7 files changed, 45 insertions(+), 26 deletions(-) diff --git a/src/server/issue_request.c b/src/server/issue_request.c index 87215a0968..71c50d2ccd 100644 --- a/src/server/issue_request.c +++ b/src/server/issue_request.c @@ -496,6 +496,7 @@ int send_request_to_remote_server( int sock = 0; char log_buf[LOCAL_LOG_BUF_SIZE]; struct tcp_chan *chan = NULL; + int old_state = PTHREAD_CANCEL_ENABLE; pthread_mutex_lock(connection[conn].ch_mutex); sock = connection[conn].ch_socket; @@ -503,13 +504,14 @@ int send_request_to_remote_server( request->rq_conn = sock; - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); if ((chan = DIS_tcp_setup(sock)) == NULL) { log_err(PBSE_MEM_MALLOC, __func__, "Could not allocate memory for socket buffer"); close_conn(sock, FALSE); + pthread_setcancelstate(old_state, NULL); return(PBSE_MEM_MALLOC); } @@ -774,7 +776,7 @@ int send_request_to_remote_server( if (close_handle == true) svr_disconnect(conn); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(rc); } /* END send_request_to_remote_server() */ diff --git a/src/server/job_func.c b/src/server/job_func.c index 08fe20f5bd..dbc47de458 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -1145,7 +1145,10 @@ int create_and_queue_array_subjob( int newstate; int newsub; int rc = PBSE_NONE; + int old_state = PTHREAD_CANCEL_ENABLE; std::string arrayid = pa->ai_qs.parent_id; + + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); template_job_mgr.lock(); pjobclone = job_clone(template_job, pa, index, place_hold); @@ -1154,11 +1157,13 @@ int create_and_queue_array_subjob( if (pjobclone == NULL) { log_err(-1, __func__, "unable to clone job in job_clone_wt"); + pthread_setcancelstate(old_state, NULL); return(NONFATAL_ERROR); } else if (pjobclone == (job *)1) { /* this happens if we attempted to clone an existing job */ + pthread_setcancelstate(old_state, NULL); return(PBSE_NONE); } @@ -1193,10 +1198,12 @@ int create_and_queue_array_subjob( if ((pa = get_array(arrayid.c_str())) == NULL) { + pthread_setcancelstate(old_state, NULL); return(FATAL_ERROR); } array_mgr.mark_as_locked(); + pthread_setcancelstate(old_state, NULL); return(NONFATAL_ERROR); } @@ -1208,6 +1215,7 @@ int create_and_queue_array_subjob( /* pjobclone has been released. No mutex left to unlock */ clone_mgr.set_unlock_on_exit(false); } + pthread_setcancelstate(old_state, NULL); return(FATAL_ERROR); } @@ -1378,10 +1386,11 @@ void *job_clone_wt( job *template_job; char *jobid; int rc; + int old_state = PTHREAD_CANCEL_ENABLE; char namebuf[MAXPATHLEN]; job_array *pa; - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); std::string prev_job_id; std::string adjusted_path_jobs; @@ -1391,7 +1400,7 @@ void *job_clone_wt( if (jobid == NULL) { log_err(ENOMEM, __func__, "Can't malloc"); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } @@ -1403,7 +1412,7 @@ void *job_clone_wt( if (template_job != NULL) unlock_ji_mutex(template_job, __func__, "1", LOGLEVEL); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } @@ -1415,7 +1424,7 @@ void *job_clone_wt( // Do not create the array sub-jobs until the job has been routed set_task(WORK_Timed, time(NULL) + ARRAY_ROUTING_RETRY_SECONDS, job_clone_task_wrapper, strdup(jobid), FALSE); free(jobid); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } @@ -1448,7 +1457,7 @@ void *job_clone_wt( if (rc == FATAL_ERROR) { - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } @@ -1462,7 +1471,7 @@ void *job_clone_wt( perform_array_postprocessing(pa); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } /* END job_clone_wt */ diff --git a/src/server/job_recov.c b/src/server/job_recov.c index b226068119..0e13f3cdc3 100644 --- a/src/server/job_recov.c +++ b/src/server/job_recov.c @@ -1203,7 +1203,8 @@ int job_save( int mom_port) /* if 0 ignore otherwise append to end of job name. this is for multi-mom mode */ { - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + int old_state = PTHREAD_CANCEL_ENABLE; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); char namebuf1[MAXPATHLEN]; char namebuf2[MAXPATHLEN]; @@ -1282,11 +1283,11 @@ int job_save( { log_event(PBSEVENT_ERROR | PBSEVENT_SECURITY, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, "call to saveJobToXML in job_save failed"); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return -1; } - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(PBSE_NONE); } /* END job_save() */ diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 1f5fe5f0ab..344dc3fabf 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -1002,8 +1002,6 @@ int parse_job_information_from_legacy_format( { free(raw_job_data); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); - throw (int)PBSE_NODE_DELETED; } } @@ -1095,8 +1093,6 @@ void process_job_info_from_json( // re-lock the node if ((np = find_nodebyname(node_name.c_str())) == NULL) { - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); - throw (int)PBSE_NODE_DELETED; } @@ -1163,6 +1159,7 @@ void *sync_node_jobs( bool sync = sji->sync_jobs; std::string node_name(sji->node_name); std::string raw_job_info(sji->job_info); + int old_state = PTHREAD_CANCEL_ENABLE; delete sji; @@ -1171,7 +1168,7 @@ void *sync_node_jobs( return(NULL); } - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); std::vector job_id_list; std::string job_list_str("jobs="); @@ -1207,7 +1204,7 @@ void *sync_node_jobs( np->unlock_node(__func__, NULL, LOGLEVEL); } - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } } @@ -1220,7 +1217,7 @@ void *sync_node_jobs( log_err(-1, __func__, log_buf); np->unlock_node(__func__, NULL, LOGLEVEL); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } @@ -1233,7 +1230,7 @@ void *sync_node_jobs( np->unlock_node(__func__, NULL, LOGLEVEL); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } /* END sync_node_jobs() */ diff --git a/src/server/reply_send.c b/src/server/reply_send.c index f20385c19c..068799efa0 100644 --- a/src/server/reply_send.c +++ b/src/server/reply_send.c @@ -95,8 +95,9 @@ static int dis_reply_write( int rc = PBSE_NONE; char log_buf[LOCAL_LOG_BUF_SIZE]; struct tcp_chan *chan = NULL; + int old_state = PTHREAD_CANCEL_ENABLE; - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); /* setup for DIS over tcp */ if ((chan = DIS_tcp_setup(sfds)) != NULL) @@ -116,7 +117,7 @@ static int dis_reply_write( DIS_tcp_cleanup(chan); } - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(rc); } /* END dis_reply_write() */ diff --git a/src/server/req_deletearray.c b/src/server/req_deletearray.c index d6d49d21ea..a911f75373 100644 --- a/src/server/req_deletearray.c +++ b/src/server/req_deletearray.c @@ -146,6 +146,8 @@ int req_deletearray( struct work_task *ptask; char log_buf[LOCAL_LOG_BUF_SIZE]; + static const int MAX_DELETE_WAIT = 3; + int wait_tries = 0; int num_skipped = 0; char owner[PBS_MAXUSER + 1]; time_t time_now = time(NULL); @@ -165,9 +167,15 @@ int req_deletearray( (pa->ai_qs.num_cloned != pa->ai_qs.num_jobs)) && (pa->ai_qs.num_cloned > 0)) { - unlock_ai_mutex(pa, __func__, NULL, 10); - sleep(1); - pa = get_array(preq->rq_ind.rq_delete.rq_objname); + if (wait_tries == MAX_DELETE_WAIT) + break; + else + { + unlock_ai_mutex(pa, __func__, NULL, 10); + sleep(1); + pa = get_array(preq->rq_ind.rq_delete.rq_objname); + wait_tries++; + } } if (pa == NULL) diff --git a/src/server/req_jobobit.c b/src/server/req_jobobit.c index a7d500471a..3a5122c7f4 100644 --- a/src/server/req_jobobit.c +++ b/src/server/req_jobobit.c @@ -2019,6 +2019,7 @@ void on_job_exit( job *pjob; int type = WORK_Deferred_Reply; char log_buf[LOCAL_LOG_BUF_SIZE]; + int old_state = PTHREAD_CANCEL_ENABLE; if (preq == NULL) type = WORK_Immed; @@ -2047,7 +2048,7 @@ void on_job_exit( return; } - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); mutex_mgr job_mutex(pjob->ji_mutex, true); job_mutex.set_unlock_on_exit(false); @@ -2183,7 +2184,7 @@ void on_job_exit( if (job_id != NULL) free(job_id); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return; } /* END on_job_exit() */ From 651d931053c4e4486a88e7c5de425ab2d3297a30 Mon Sep 17 00:00:00 2001 From: Jo Vandeginste Date: Mon, 24 Apr 2017 12:01:13 +0200 Subject: [PATCH 235/312] Support the spool option when making rpm The --[en|dis]able-spool parameter was not passed to the rpmbuild phase, which meant the spool setting was not altered when making the rpm. This patch solves this issue. --- configure.ac | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index d4a55836cc..94c92c9cd7 100644 --- a/configure.ac +++ b/configure.ac @@ -1984,10 +1984,10 @@ AC_ARG_ENABLE(spool, [ $TORQUEHOME/spool and copy them to the users home directory when the job completes.], [case "${enableval}" in - yes) NO_SPOOL_OUTPUT=0 ;; - no) NO_SPOOL_OUTPUT=1 ;; + yes) NO_SPOOL_OUTPUT=0 ; RPM_AC_OPTS="$RPM_AC_OPTS --with spool" ;; + no) NO_SPOOL_OUTPUT=1 ; RPM_AC_OPTS="$RPM_AC_OPTS --without spool" ;; *) AC_MSG_ERROR(--enable-spool cannot take a value) ;; -esac],[NO_SPOOL_OUTPUT=0])dnl +esac],[NO_SPOOL_OUTPUT=0 ; RPM_AC_OPTS="$RPM_AC_OPTS --with spool"])dnl AC_DEFINE_UNQUOTED(NO_SPOOL_OUTPUT, ${NO_SPOOL_OUTPUT}, [directly use homedirs instead of $TORQUEHOME/spool]) AC_ARG_ENABLE(shell-use-argv, [ From 97c81b23fc8d3d51f044076ecad4a49426386afd Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 25 Apr 2017 11:32:54 -0600 Subject: [PATCH 236/312] TRQ-3995. Take the first step towards being able to parse RUR for usage information on the Cray. --- configure.ac | 1 + src/resmom/Makefile.am | 2 +- src/resmom/cray_taskstats.cpp | 103 ++++++++++++++++++++++++++ src/test/Makefile.am | 2 +- src/test/cray_taskstats/Makefile.am | 3 + src/test/cray_taskstats/scaffolding.c | 88 ++++++++++++++++++++++ src/test/cray_taskstats/test_uut.c | 95 ++++++++++++++++++++++++ 7 files changed, 292 insertions(+), 2 deletions(-) create mode 100644 src/resmom/cray_taskstats.cpp create mode 100644 src/test/cray_taskstats/Makefile.am create mode 100644 src/test/cray_taskstats/scaffolding.c create mode 100644 src/test/cray_taskstats/test_uut.c diff --git a/configure.ac b/configure.ac index 94c92c9cd7..2b9a3a6adc 100644 --- a/configure.ac +++ b/configure.ac @@ -486,6 +486,7 @@ m4_ifdef([AM_SILENT_RULES],[ src/test/checkpoint/Makefile src/test/cray_cpa/Makefile src/test/cray_energy/Makefile + src/test/cray_taskstats/Makefile src/test/generate_alps_status/Makefile src/test/mom_job_func/Makefile src/test/mom_comm/Makefile diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index 06943b304b..18a9aabcf9 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -44,7 +44,7 @@ pbs_mom_SOURCES = catch_child.c mom_comm.c mom_inter.c mom_main.c \ release_reservation.c generate_alps_status.c \ parse_config.c node_frequency.cpp cray_energy.c \ accelerators_numa.cpp pmix_interface.c pmix_tracker.cpp \ - pmix_operation.cpp \ + pmix_operation.cpp cray_taskstats.cpp \ ../server/attr_recov.c ../server/dis_read.c \ ../server/job_attr_def.c ../server/job_recov.c \ ../server/reply_send.c ../server/resc_def_all.c \ diff --git a/src/resmom/cray_taskstats.cpp b/src/resmom/cray_taskstats.cpp new file mode 100644 index 0000000000..0d6b32a1ec --- /dev/null +++ b/src/resmom/cray_taskstats.cpp @@ -0,0 +1,103 @@ + +#include +#include +#include +#include + +#include "pbs_job.h" +#include "resource.h" +#include "pbs_error.h" +#include "resmon.h" + +/* + * parse_rur_line() + * + * Parses a line of an rur file to find the stats for this job. + * @param line - the line of the file + * @param pjob - the job whose stats we're finding + */ + +// RUR usage lines look like this +// 2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 10000000, 'stime', 0, 'max_rss', 940, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] + + +int parse_rur_line( + + const std::string &line, + job *pjob) + + { + size_t pos = line.find("taskstats"); + + if (pos != std::string::npos) + { + if (line.at(pos) == ' ') + pos++; + if (line.at(pos) == '[') + pos++; + + std::string stats = line.substr(pos, std::string::npos); + char *work_str = strdup(stats.c_str()); + char *ptr = strstr(work_str, "utime',"); + long cput = 0; + long long mem = 0; + resource_def *rd; + resource *pres; + + if (ptr != NULL) + { + ptr += 7; // move past "utime'," + + if (*ptr == ' ') + ptr++; + + cput = strtol(ptr, &ptr, 10); + } + + if ((ptr = strstr(ptr, "stime',")) != NULL) + { + ptr += 7; // move past "stime'," + + if (*ptr == ' ') + ptr++; + + cput += strtol(ptr, &ptr, 10); + } + + if (cput > 0) + { + rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); + pres = add_resource_entry(pjob->ji_wattr + JOB_ATR_resc_used, rd); + pres->rs_value.at_flags |= ATR_VFLAG_SET; + pres->rs_value.at_type = ATR_TYPE_LONG; + pres->rs_value.at_val.at_long = MAX(cput, pres->rs_value.at_val.at_long); + } + + if ((ptr = strstr(ptr, "max_rss',")) != NULL) + { + ptr += 9; // move past "max_rss'," + + if (*ptr == ' ') + ptr++; + + mem = strtoll(ptr, NULL, 10); + + if (mem > 0) + { + rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); + pres = add_resource_entry(pjob->ji_wattr + JOB_ATR_resc_used, rd); + pres->rs_value.at_flags |= ATR_VFLAG_SET; + pres->rs_value.at_type = ATR_TYPE_SIZE; + pres->rs_value.at_val.at_size.atsv_shift = 10; + pres->rs_value.at_val.at_size.atsv_units = ATR_SV_BYTESZ; + pres->rs_value.at_val.at_size.atsv_num = MAX(mem, pres->rs_value.at_val.at_size.atsv_num); + } + } + + free(work_str); + } + + return(PBSE_NONE); + } // parse_rur_line() + + diff --git a/src/test/Makefile.am b/src/test/Makefile.am index 52194066e6..5824f977cf 100644 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -68,7 +68,7 @@ CMDS_UT_DIRS = MXML common_cmds pbs_track pbsdsh pbsnodes pbspd pbspoe qalter qc MISC_UT_DIRS = momctl -MOM_UT_DIRS = alps_reservations catch_child checkpoint cray_energy generate_alps_status \ +MOM_UT_DIRS = alps_reservations catch_child checkpoint cray_energy cray_taskstats generate_alps_status \ mom_comm mom_inter mom_job_func mom_mach mom_main mom_process_request mom_req_quejob \ mom_server mom_start parse_config pbs_demux prolog release_reservation requests \ start_exec tmsock_recov diff --git a/src/test/cray_taskstats/Makefile.am b/src/test/cray_taskstats/Makefile.am new file mode 100644 index 0000000000..c64a3ab0cf --- /dev/null +++ b/src/test/cray_taskstats/Makefile.am @@ -0,0 +1,3 @@ +include ../Makefile_Mom.ut + +libuut_la_SOURCES = ${PROG_ROOT}/cray_taskstats.cpp diff --git a/src/test/cray_taskstats/scaffolding.c b/src/test/cray_taskstats/scaffolding.c new file mode 100644 index 0000000000..c9ad0bb42c --- /dev/null +++ b/src/test/cray_taskstats/scaffolding.c @@ -0,0 +1,88 @@ +#include "license_pbs.h" /* See here for the software license */ +#include +#include +#include + +#include "attribute.h" +#include "resource.h" + +resource_def *svr_resc_def; +int svr_resc_size = 0; + +int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ + +void log_err(int errnum, const char *routine, const char *text) + { + } + +void log_event(int eventtype, int objclass, const char *objname, const char *text) + { + } + +void init_attr(pbs_attribute *pattr) + { + memset(pattr, 0, sizeof(pbs_attribute)); + } + +resource_def *find_resc_def( + + resource_def *rscdf, /* address of array of resource_def structs */ + const char *name, /* name of resource */ + int limit) /* number of members in resource_def array */ + + { + static resource_def mem_def; + static resource_def cput_def; + + if (!strcmp(name, "mem")) + { + mem_def.rs_name = name; + mem_def.rs_free = init_attr; + return(&mem_def); + } + else + { + cput_def.rs_name = name; + cput_def.rs_free = init_attr; + return(&cput_def); + } + } + + +resource *add_resource_entry( + + pbs_attribute *pattr, + resource_def *prdef) + + { + resource new_resource; + + if (pattr->at_val.at_ptr == NULL) + pattr->at_val.at_ptr = new std::vector(); + + std::vector *resources = (std::vector *)pattr->at_val.at_ptr; + + for (size_t i = 0; i < resources->size(); i++) + { + resource &r = resources->at(i); + + if (!strcmp(r.rs_defin->rs_name, prdef->rs_name)) + { + return(&r); + } + } + + new_resource.rs_defin = prdef; + new_resource.rs_value.at_type = prdef->rs_type; + new_resource.rs_value.at_flags = 0; + prdef->rs_free(&new_resource.rs_value); + + resources->push_back(new_resource); + pattr->at_flags |= ATR_VFLAG_SET | ATR_VFLAG_MODIFY; + + // Return the element we just added + resource &r = resources->at(resources->size() - 1); + + return(&r); + } /* END add_resource_entry() */ + diff --git a/src/test/cray_taskstats/test_uut.c b/src/test/cray_taskstats/test_uut.c new file mode 100644 index 0000000000..c08563472b --- /dev/null +++ b/src/test/cray_taskstats/test_uut.c @@ -0,0 +1,95 @@ +#include "license_pbs.h" /* See here for the software license */ +#include +#include +#include + +#include "pbs_error.h" +#include "pbs_job.h" +#include "resource.h" + +#include + +int parse_rur_line(const std::string &line, job *pjob); +const char *l1 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 100, 'stime', 100, 'max_rss', 1984, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; +const char *l2 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 300, 'stime', 0, 'max_rss', 4444, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; + +START_TEST(test_parse_rur_line) + { + job pjob; + + memset(&pjob, 0, sizeof(pjob)); + fail_unless(parse_rur_line(l1, &pjob) == PBSE_NONE); + resource_def *rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); + resource *pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_long == 200); + + rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_size.atsv_num == 1984, + "%llu", pres->rs_value.at_val.at_size.atsv_num); + + // Make sure we increase the value if we are using more later + fail_unless(parse_rur_line(l2, &pjob) == PBSE_NONE); + rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_long == 300); + + rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_size.atsv_num == 4444); + + // Make sure the values don't decrease if we find less later + fail_unless(parse_rur_line(l1, &pjob) == PBSE_NONE); + rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_long == 300); + + rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_size.atsv_num == 4444); + } +END_TEST + +START_TEST(test_two) + { + + + } +END_TEST + +Suite *cray_taskstats_suite(void) + { + Suite *s = suite_create("cray_taskstats_suite methods"); + TCase *tc_core = tcase_create("test_parse_rur_line"); + tcase_add_test(tc_core, test_parse_rur_line); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_two"); + tcase_add_test(tc_core, test_two); + suite_add_tcase(s, tc_core); + + return s; + } + +void rundebug() + { + } + +int main(void) + { + int number_failed = 0; + SRunner *sr = NULL; + rundebug(); + sr = srunner_create(cray_taskstats_suite()); + srunner_set_log(sr, "cray_taskstats_suite.log"); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return number_failed; + } From 41bc4a5e828531c60b77e1a31636ad1d81cce211 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 25 Apr 2017 11:34:38 -0600 Subject: [PATCH 237/312] TRQ-3986. Add a check before erasing an object from container. --- src/include/container.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/include/container.hpp b/src/include/container.hpp index 8fa9e9d0b1..0b4e46ca3a 100644 --- a/src/include/container.hpp +++ b/src/include/container.hpp @@ -816,7 +816,9 @@ class item_container int prev = slots[index].prev; int next = slots[index].next; - map.erase(slots[index].pItem->id); + if (map.find(slots[index].pItem->id) != map.end()) + map.erase(slots[index].pItem->id); + slots[index].prev = ALWAYS_EMPTY_INDEX; slots[index].next = ALWAYS_EMPTY_INDEX; delete slots[index].pItem; From 205404529421f3ac5e8e22f173cf211a4508acbe Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 25 Apr 2017 15:53:44 -0600 Subject: [PATCH 238/312] TRQ-4015. Remove : from the server_name file when adding the server name to a job id. --- src/lib/Libcmds/get_server.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/Libcmds/get_server.c b/src/lib/Libcmds/get_server.c index 643c979685..e4e2ec57a8 100644 --- a/src/lib/Libcmds/get_server.c +++ b/src/lib/Libcmds/get_server.c @@ -158,6 +158,11 @@ int get_server_and_job_ids( if (notNULL(parent_server)) { + char *colon = strchr(parent_server, ':'); + + if (colon != NULL) + *colon = '\0'; + snprintf(def_server, PBS_MAXSERVERNAME, "%s", parent_server); c = def_server; From 08aa6c59be17579435ecf6dde4c064733dd6f881 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 26 Apr 2017 09:23:28 -0600 Subject: [PATCH 239/312] TRQ-3989 - Preserve node gpu counts across pbs_server restarts --- src/server/node_manager.c | 104 +++++++++++++++++++++++++++- src/test/node_manager/scaffolding.c | 20 +++++- src/test/node_manager/test_uut.c | 38 ++++++++++ 3 files changed, 159 insertions(+), 3 deletions(-) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 344dc3fabf..42138076af 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -3698,7 +3698,8 @@ int add_job_to_gpu_subnode( pnode->nd_ngpus_free--; gn.inuse = true; gn.job_count++; - pnode->nd_ngpus_to_be_used--; + if (pnode->nd_ngpus_to_be_used > 0) + pnode->nd_ngpus_to_be_used--; return(PBSE_NONE); } /* END add_job_to_gpu_subnode() */ @@ -5872,6 +5873,73 @@ int set_one_old( } /* END set_one_old() */ +/* + * Process gpu token of the form -gpu/[-] + * + * Set gpu subjob information for node. + */ + +int process_gpu_token( + + const char *gpu_token, + job *pjob) + + { + char *pc; + char *dash; + char *p; + int first; + int last; + struct pbsnode *pnode; + + // gpu_token expected to point to something like "numa3-gpu/2" + + if ((gpu_token == NULL) || (pjob == NULL)) + return(-1); + + // calculate range indices after the / + if ((pc = strchr((char *)gpu_token, (int)'/'))) + { + first = strtol(pc + 1, &dash, 10); + + *pc = '\0'; + + if (*dash == '-') + last = strtol(dash + 1, NULL, 10); + else + last = first; + } + else + { + first = 0; + last = first; + } + + // drop -gpu suffix + if ((p = strrchr((char *)gpu_token, (int)'-')) != NULL) + { + if (strcmp(p, "-gpu") == 0) + *p = '\0'; + } + + // lookup node and set gpu info on each gpu subnode + if ((pnode = find_nodebyname(gpu_token)) != NULL) + { + int i; + + for (i = first; i <= last; i++) + { + gpusubn &gn = pnode->nd_gpusn[i]; + + add_job_to_gpu_subnode(pnode, gn, pjob); + } + + // unlock node + pnode->unlock_node(__func__, NULL, LOGLEVEL); + } + + return(PBSE_NONE); + } /* * set_old_nodes - set "old" nodes as in use - called from pbsd_init() @@ -5887,6 +5955,40 @@ int set_old_nodes( char *po; int rc = PBSE_NONE; + // handle gpu info + if (pjob->ji_wattr[JOB_ATR_exec_gpus].at_flags & ATR_VFLAG_SET) + { + char *old_str; + + if ((old_str = strdup(pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str)) == NULL) + { + return(PBSE_SYSTEM); + } + + while ((po = strrchr(old_str, (int)'+')) != NULL) + { + // remove + + *po++ = '\0'; + + if (process_gpu_token(po, pjob) != PBSE_NONE) + { + free(old_str); + return(PBSE_SYSTEM); + } + } + + if (old_str != NULL) + { + if (process_gpu_token(old_str, pjob) != PBSE_NONE) + { + free(old_str); + return(PBSE_SYSTEM); + } + } + + free(old_str); + } + if (pjob->ji_wattr[JOB_ATR_exec_host].at_flags & ATR_VFLAG_SET) { old = strdup(pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str); diff --git a/src/test/node_manager/scaffolding.c b/src/test/node_manager/scaffolding.c index 8d03fc30a2..f76232afc9 100644 --- a/src/test/node_manager/scaffolding.c +++ b/src/test/node_manager/scaffolding.c @@ -57,6 +57,7 @@ bool job_mode = false; int can_place = 0; pbsnode napali_node; +int create_a_gpusubnode(struct pbsnode*); struct batch_request *alloc_br(int type) { @@ -134,11 +135,20 @@ struct pbsnode *find_nodebyname(const char *nodename) { static struct pbsnode bob; static struct pbsnode other; + static struct pbsnode gpunode; static int called = 0; if (called == 0) { other.change_name("lihue"); + + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + called++; } @@ -153,9 +163,9 @@ struct pbsnode *find_nodebyname(const char *nodename) else if (!strcmp(nodename, "3")) return(&bob); else if (!strcmp(nodename, "lihue")) - { return(&other); - } + else if (!strcmp(nodename, "gpunode")) + return(&gpunode); else return(NULL); } @@ -537,6 +547,12 @@ pbs_net_t get_hostaddr( int create_a_gpusubnode(struct pbsnode *np) { + gpusubn tmp(np->nd_gpusn.size()); + + np->nd_gpusn.push_back(tmp); + np->nd_ngpus++; + np->nd_ngpus_free++; + return(0); } diff --git a/src/test/node_manager/test_uut.c b/src/test/node_manager/test_uut.c index bb8933d994..5d224b7f5c 100644 --- a/src/test/node_manager/test_uut.c +++ b/src/test/node_manager/test_uut.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "license_pbs.h" /* See here for the software license */ @@ -46,6 +47,7 @@ void update_failure_counts(const char *node_name, int rc); void check_node_jobs_existence(struct work_task *pwt); int add_job_to_gpu_subnode(pbsnode *pnode, gpusubn &gn, job *pjob); int proplist(char **str, std::vector &plist, int *node_req, int *gpu_req, int *mic_req); +int process_gpu_token(const char*, job*); @@ -953,6 +955,38 @@ START_TEST(place_subnodes_in_hostlist_job_exclusive_test) } END_TEST +START_TEST(test_process_gpu_token) + { + job *pjob; + char *s; + struct pbsnode *pnode; + + pjob = (job *)calloc(1, sizeof(job)); + s = strdup("gpunode/5"); + + fail_unless(process_gpu_token(NULL, pjob) != PBSE_NONE); + fail_unless(process_gpu_token(s, NULL) != PBSE_NONE); + fail_unless(process_gpu_token(NULL, NULL) != PBSE_NONE); + + pjob->ji_internal_id = 10; + fail_unless(process_gpu_token(s, pjob) == PBSE_NONE); + + fail_unless((pnode = find_nodebyname("gpunode")) != NULL); + + fail_unless(pnode->nd_gpusn[5].job_internal_id == 10); + fail_unless(pnode->nd_gpusn[5].inuse == true); + fail_unless(pnode->nd_gpusn[5].job_count == 1); + + s = strdup("gpunode/0-2"); + fail_unless(process_gpu_token(s, pjob) == PBSE_NONE); + fail_unless(pnode->nd_gpusn[0].inuse == true); + fail_unless(pnode->nd_gpusn[0].job_count == 1); + fail_unless(pnode->nd_gpusn[1].inuse == true); + fail_unless(pnode->nd_gpusn[1].job_count == 1); + fail_unless(pnode->nd_gpusn[2].inuse == true); + fail_unless(pnode->nd_gpusn[2].job_count == 1); + } +END_TEST Suite *node_manager_suite(void) { @@ -1016,6 +1050,10 @@ Suite *node_manager_suite(void) tcase_add_test(tc_core, check_node_jobs_exitence_test); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_process_gpu_token"); + tcase_add_test(tc_core, test_process_gpu_token); + suite_add_tcase(s, tc_core); + return(s); } From 25bc0f4504bcde89512b4186bf770f6cc19d81bd Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 27 Apr 2017 13:39:48 -0600 Subject: [PATCH 240/312] TRQ-3627. Make resources_max.proc work correctly. Conflicts: src/server/svr_jobfunc.c --- src/server/svr_jobfunc.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index cac322ffb3..2bbc48c83d 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -1775,14 +1775,6 @@ int chk_svr_resc_limit( } } } - - /* Added 6/14/2010 Ken Nielson for ability to parse the procs resource */ - else if ((r.rs_defin == procresc) && - (qtype == QTYPE_Execution)) - { - proc_ct = r.rs_value.at_val.at_long; - } - #ifdef NERSCDEV else if (r.rs_defin == mppwidthresc) { @@ -1809,6 +1801,11 @@ int chk_svr_resc_limit( (r.rs_defin != needresc)) { /* don't check neednodes */ + if ((r.rs_defin == procresc) && + (qtype == QTYPE_Execution)) + { + proc_ct = r.rs_value.at_val.at_long; + } rc = r.rs_defin->rs_comp(&cmpwith->rs_value, &r.rs_value); From f022946165405ba15e4a80e510c3f0264e2ae68e Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 27 Apr 2017 13:46:42 -0600 Subject: [PATCH 241/312] TRQ-3413. Make trqauthd the default Authentication method displayed by configure. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 2b9a3a6adc..7d751f37a7 100644 --- a/configure.ac +++ b/configure.ac @@ -1718,7 +1718,7 @@ case "$jobcreatedir" in esac AC_MSG_RESULT([$jobcreatedir]) -AUTH_TYPE="classic (pbs_iff)" +AUTH_TYPE="trqauthd" dnl compile for munge authorization - allows sites to use munge dnl as a means to validate user identity from remote hosts. (executable based implementation) From 8c915d786c0eb3d2934fad518e0321813ff8b2b6 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 27 Apr 2017 16:13:06 -0600 Subject: [PATCH 242/312] TRQ-3722. Add the option --with-reserved-port-start to configure This allows the site to set a different reserved port starting point from 144 if desired. --- configure.ac | 32 ++++++++++++++++++++++++++++++++ src/lib/Libnet/net_client.c | 22 ++++++++++------------ 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/configure.ac b/configure.ac index 7d751f37a7..8d785952da 100644 --- a/configure.ac +++ b/configure.ac @@ -2128,6 +2128,38 @@ AC_DEFINE_UNQUOTED(TCP_RETRY_LIMIT,${TCP_RETRY_LIMIT},[Define to set a max retry +AC_ARG_WITH(reserved_port_start, [ + --with-reserved-port-start=NUMBER + Set the starting reserved port number to be used. Must be a number + greater than 143 and less than 824. Please be careful when setting this, + as restricting the number too much may not leave enough privileged ports + available to pbs_server, which can potential slow down the server + immensely], + [RESERVED_PORT_START=${withval}],[RESERVED_PORT_START=144]) + +dnl Make sure it's a completely numeric string +if [[ $(expr "x${RESERVED_PORT_START}" : "x[0-9]*$") -gt 0 ]]; then + echo "Setting reserved port to get ports from ${RESERVED_PORT_START} to 1023 inclusive." +else + echo "Value ${RESERVER_PORT_START} is not permitted for --with-reserved-port-start because it contains non-numeric characters." + exit -1 +fi + +dnl Make sure it isn't less than 144 +if [[ $RESERVED_PORT_START -lt 144 ]]; then + echo "Value ${RESERVED_PORT_START} is not permitted for --with-reserved-port-start because it is less than 144" + exit -1 +fi + +dnl Make sure it's less than 824 so that there are at least 200 reserved ports available for the daemons. +if [[ $RESERVED_PORT_START -gt 823 ]]; then + echo "Value ${RESERVED_PORT_START} is not permitted for --with-reserved-port-start because it is greater than 823 and we need a range of at least 200 possible privileged ports." + exit -1 +fi +AC_DEFINE_UNQUOTED(RESERVED_PORT_START, ${RESERVED_PORT_START},[Define to specify a different reserved port starting point]) + + + AC_ARG_WITH(default_server, [ --with-default-server=HOSTNAME set the name of the computer that clients will diff --git a/src/lib/Libnet/net_client.c b/src/lib/Libnet/net_client.c index 0dff89d684..8f8542320b 100644 --- a/src/lib/Libnet/net_client.c +++ b/src/lib/Libnet/net_client.c @@ -236,7 +236,6 @@ int client_to_svr( char *EMsg) /* O (optional,minsize=1024) */ { - const char id[] = "client_to_svr"; struct sockaddr_in local; struct sockaddr_in remote; int sock; @@ -252,9 +251,8 @@ int client_to_svr( int trycount = 0; struct timespec rem; -#define STARTPORT 144 #define ENDPORT (IPPORT_RESERVED - 1) -#define NPORTS (ENDPORT - STARTPORT + 1) +#define NPORTS (ENDPORT - RESERVED_PORT_START + 1) #define SHUFFLE_COUNT 3 if (EMsg != NULL) @@ -290,7 +288,7 @@ int client_to_svr( if (EMsg != NULL) sprintf(EMsg, "cannot create socket in %s - errno: %d %s", - id, + __func__, errno, err_buf); @@ -301,7 +299,7 @@ int client_to_svr( { if (EMsg != NULL) sprintf(EMsg, "PBS_NET_MAX_CONNECTIONS exceeded in %s", - id); + __func__); close(sock); /* too many connections */ @@ -364,7 +362,7 @@ int client_to_svr( /* bindresvport could not get a privileged port */ if (EMsg != NULL) sprintf(EMsg, "cannot bind to reserved port in %s - errno: %d %s", - id, + __func__, errno, err_buf); @@ -379,7 +377,7 @@ int client_to_svr( #else /* HAVE_BINDRESVPORT */ /* Pseudo-casual shuffling of tryport */ - tryport = (rand() % NPORTS) + STARTPORT; + tryport = (rand() % NPORTS) + RESERVED_PORT_START; #endif /* HAVE_BINDRESVPORT */ } @@ -389,7 +387,7 @@ int client_to_svr( if (tryport > ENDPORT) { - tryport = STARTPORT; + tryport = RESERVED_PORT_START; } } @@ -434,7 +432,7 @@ int client_to_svr( err_buf[0] = '\0'; sprintf(EMsg, "cannot bind to reserved port in %s - errno: %d %s", - id, + __func__, errno, err_buf); } @@ -506,7 +504,7 @@ int client_to_svr( if (EMsg != NULL) sprintf(EMsg, "cannot connect to port %d in %s - connection refused.\n Check if trqauthd should be running\n", port, - id); + __func__); close(sock); @@ -566,7 +564,7 @@ int client_to_svr( sprintf(EMsg, "cannot connect to port %d in %s - errno:%d %s", port, - id, + __func__, errno, err_buf); } @@ -595,7 +593,7 @@ int client_to_svr( if (EMsg != NULL) sprintf(EMsg, "cannot connect to port %d in %s - errno:%d %s", tryport, - id, + __func__, errno, err_buf); From 0b9d7b6d686f62e8c5faa9d31eb756b10113ba97 Mon Sep 17 00:00:00 2001 From: David Beer Date: Fri, 28 Apr 2017 11:23:22 -0600 Subject: [PATCH 243/312] TRQ-3577. Make notes for cray compute nodes last across restarts. Conflicts: src/test/node_func/scaffolding.c --- src/server/node_func.c | 3 ++- src/server/node_manager.c | 14 ++++++++------ src/test/node_func/scaffolding.c | 8 ++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/server/node_func.c b/src/server/node_func.c index 8d41c3d4ca..ebd3304f4f 100644 --- a/src/server/node_func.c +++ b/src/server/node_func.c @@ -114,6 +114,7 @@ job *get_job_from_job_usage_info(job_usage_info *jui, struct pbsnode *pnode); +pbsnode *get_compute_node(const char *node_name); struct pbsnode *alps_reporter; @@ -2001,7 +2002,7 @@ void load_node_notes() (cray_enabled == true) && (isdigit(node_name.at(0)))) { - pnode = create_alps_subnode(alps_reporter, node_name.c_str()); + pnode = get_compute_node(node_name.c_str()); } if (pnode != NULL) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 42138076af..446bb765cc 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -1718,8 +1718,8 @@ void write_node_power_state(void) int write_node_note(void) { - struct pbsnode *np; - all_nodes_iterator *iter = NULL; + struct pbsnode *np = NULL; + node_iterator iter; FILE *nin; if (LOGLEVEL >= 2) @@ -1739,9 +1739,11 @@ int write_node_note(void) return(-1); } + + reinitialize_node_iterator(&iter); /* for each node ... */ - while ((np = next_host(&allnodes, &iter, NULL)) != NULL) + while ((np = next_node(&allnodes, np, &iter)) != NULL) { /* write node name followed by its note string */ if (np->nd_note.size() != 0) @@ -1752,8 +1754,8 @@ int write_node_note(void) np->unlock_node(__func__, NULL, LOGLEVEL); } - if (iter != NULL) - delete iter; + if (iter.node_index != NULL) + delete iter.node_index; fflush(nin); @@ -5739,7 +5741,7 @@ void free_nodes( struct pbsnode *get_compute_node( - char *node_name) + const char *node_name) { struct pbsnode *ar = alps_reporter; diff --git a/src/test/node_func/scaffolding.c b/src/test/node_func/scaffolding.c index e1cf4600f2..a6206b7ef5 100644 --- a/src/test/node_func/scaffolding.c +++ b/src/test/node_func/scaffolding.c @@ -598,3 +598,11 @@ bool authorized_hosts::remove_address(unsigned long addr, unsigned short port) authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; + +struct pbsnode *get_compute_node( + + const char *node_name) + + { + return(NULL); + } From af1c6e216f47bd9b8b107a1446305717ef8664fd Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Mon, 1 May 2017 11:39:34 -0600 Subject: [PATCH 244/312] TRQ-3716 Cause make rpm to default to use the prefix resulting from configure --- configure.ac | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 8d785952da..8b03c5ace6 100644 --- a/configure.ac +++ b/configure.ac @@ -2599,6 +2599,10 @@ dnl AC_CREATE_GENERIC_CONFIG([pbs],[-ltorque]) +# Cause make rpm to default to use the prefix resulting from configure +RPM_AC_OPTS="--define '_prefix ${prefix}'$RPM_AC_OPTS" +AC_SUBST(RPM_AC_OPTS) + AC_SUBST(ALPS_LIBS) dnl these are quoted, and this removes them @@ -2606,7 +2610,6 @@ SPEC_NAME=$PACKAGE_TARNAME SPEC_VERSION=$PACKAGE_VERSION AC_SUBST(SPEC_NAME) AC_SUBST(SPEC_VERSION) -AC_SUBST(RPM_AC_OPTS) dnl Head and bottom for the config-file AH_TOP([#ifndef _PBS_CONFIG_H_ From 14ef3c3704b08f9877c85ab280f2de936a8016a6 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 2 May 2017 09:40:50 -0600 Subject: [PATCH 245/312] TRQ-3489 - look for xauth in /usr/bin --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 8b03c5ace6..124aee8da0 100644 --- a/configure.ac +++ b/configure.ac @@ -2354,6 +2354,7 @@ AC_ARG_WITH(xauth, ], [ TestPath="$PATH" + TestPath="${TestPath}${PATH_SEPARATOR}/usr/bin" TestPath="${TestPath}${PATH_SEPARATOR}/usr/X/bin" TestPath="${TestPath}${PATH_SEPARATOR}/usr/bin/X11" TestPath="${TestPath}${PATH_SEPARATOR}/usr/X11R6/bin" From 1d06f0002b3b0cd38d2d720b66d99d6648e230d4 Mon Sep 17 00:00:00 2001 From: Isaac Tobler Date: Tue, 2 May 2017 10:27:06 -0600 Subject: [PATCH 246/312] small update to include entire range of ports for TRQ-1264 --- src/cmds/qsub_functions.c | 2 +- src/test/qsub_functions/test_uut.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index f2667814cf..33776770b2 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -1672,7 +1672,7 @@ void do_dir( unsigned int get_port_in_range() { - int span = interactive_port_max - interactive_port_min; + int span = interactive_port_max - interactive_port_min + 1; return(rand() % span + interactive_port_min); } // END get_port_in_range() diff --git a/src/test/qsub_functions/test_uut.c b/src/test/qsub_functions/test_uut.c index 46021933f6..cc206a8b3c 100644 --- a/src/test/qsub_functions/test_uut.c +++ b/src/test/qsub_functions/test_uut.c @@ -63,7 +63,7 @@ START_TEST(test_get_port_in_range) interactive_port_min = 30000; interactive_port_max = 30100; - for (int i = 0; i < 100; i++) + for (int i = 0; i <= 100; i++) { unsigned int port = get_port_in_range(); fail_unless(port >= interactive_port_min); From 6c08bd58684451819786a70e4db6855cda23efac Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 2 May 2017 11:13:24 -0600 Subject: [PATCH 247/312] TRQ-3951. Handle job dependencies in routing queues. Previously, they weren't handled until the job hit an execution queue, which causes problems if the job it depends on has finished by the time it is routed. --- src/server/req_register.c | 18 +------------- src/server/svr_jobfunc.c | 40 +++++++++++++++++--------------- src/test/req_register/test_uut.c | 22 ++++++++++++++++++ 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/src/server/req_register.c b/src/server/req_register.c index 6944c9d817..dd217b7ad6 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -1426,31 +1426,15 @@ int depend_on_que( int rc = PBSE_NONE; int type; job *pjob = (job *)pj; - pbs_queue *pque; char job_id[PBS_MAXSVRJOBID+1]; strcpy(job_id, pjob->ji_qs.ji_jobid); - pque = get_jobs_queue(&pjob); - if (pque == NULL) - { - if (pjob == NULL) - { - log_err(PBSE_JOBNOTFOUND, __func__, "Job lost while acquiring queue 8"); - return(PBSE_JOBNOTFOUND); - } - else - return(PBSE_NONE); - } - - mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true); if (((mode != ATR_ACTION_ALTER) && - (mode != ATR_ACTION_NOOP)) || - (pque->qu_qs.qu_type != QTYPE_Execution)) + (mode != ATR_ACTION_NOOP))) { return(PBSE_NONE); } - pque_mutex.unlock(); if (mode == ATR_ACTION_ALTER) { diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index 2bbc48c83d..68aac16318 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -655,25 +655,6 @@ int svr_enquejob( /* do anything needed doing regarding job dependencies */ que_mgr.unlock(); - if ((pjob->ji_qs.ji_state != JOB_STATE_COMPLETE) && - (pjob->ji_qs.ji_substate != JOB_SUBSTATE_COMPLETE) && - (pjob->ji_wattr[JOB_ATR_depend].at_flags & ATR_VFLAG_SET)) - { - try - { - rc = depend_on_que(&pjob->ji_wattr[JOB_ATR_depend], pjob, ATR_ACTION_NOOP); - } - catch (int pbs_errcode) - { - rc = pbs_errcode; - } - - if (rc == PBSE_JOBNOTFOUND) - return(rc); - else if (rc != PBSE_NONE) - rc = PBSE_BADDEPEND; - } - /* set eligible time */ if (((pjob->ji_wattr[JOB_ATR_etime].at_flags & ATR_VFLAG_SET) == 0) && (pjob->ji_qs.ji_state == JOB_STATE_QUEUED)) @@ -695,6 +676,27 @@ int svr_enquejob( /* start attempts to route job */ pjob->ji_qs.ji_un_type = JOB_UNION_TYPE_ROUTE; pjob->ji_qs.ji_un.ji_routet.ji_quetime = time_now; + + que_mgr.unlock(); + } + + if ((pjob->ji_qs.ji_state != JOB_STATE_COMPLETE) && + (pjob->ji_qs.ji_substate != JOB_SUBSTATE_COMPLETE) && + (pjob->ji_wattr[JOB_ATR_depend].at_flags & ATR_VFLAG_SET)) + { + try + { + rc = depend_on_que(&pjob->ji_wattr[JOB_ATR_depend], pjob, ATR_ACTION_NOOP); + } + catch (int pbs_errcode) + { + rc = pbs_errcode; + } + + if (rc == PBSE_JOBNOTFOUND) + return(rc); + else if (rc != PBSE_NONE) + rc = PBSE_BADDEPEND; } return(rc); diff --git a/src/test/req_register/test_uut.c b/src/test/req_register/test_uut.c index 0974332900..c9f89c9675 100644 --- a/src/test/req_register/test_uut.c +++ b/src/test/req_register/test_uut.c @@ -39,6 +39,7 @@ int req_register(batch_request *preq); bool remove_array_dependency_job_from_job(struct array_depend *pdep, job *pjob, char *job_array_id); void removeAfterAnyDependency(const char *pJobID, const char *targetJob); bool job_ids_match(const char *parent, const char *child); +int depend_on_que(pbs_attribute *pattr, void *pj, int mode); extern char server_name[]; @@ -63,6 +64,26 @@ void initialize_depend_attr( } /* END initialize_depend_attr() */ +START_TEST(test_depend_on_que) + { + job *pjob = (job *)calloc(1, sizeof(job)); + depend *pdep; + initialize_depend_attr(pjob->ji_wattr + JOB_ATR_depend); + pdep = make_depend(JOB_DEPEND_TYPE_AFTERNOTOK, pjob->ji_wattr + JOB_ATR_depend); + make_dependjob(pdep, job2); + + // These three modes should do nothing + fail_unless(depend_on_que(pjob->ji_wattr + JOB_ATR_depend, pjob, ATR_ACTION_NEW) == PBSE_NONE); + fail_unless(depend_on_que(pjob->ji_wattr + JOB_ATR_depend, pjob, ATR_ACTION_FREE) == PBSE_NONE); + fail_unless(depend_on_que(pjob->ji_wattr + JOB_ATR_depend, pjob, ATR_ACTION_RECOV) == PBSE_NONE); + + fail_unless(depend_on_que(pjob->ji_wattr + JOB_ATR_depend, pjob, ATR_ACTION_ALTER) == PBSE_NONE); + fail_unless((pjob->ji_wattr[JOB_ATR_hold].at_flags & ATR_VFLAG_SET) != 0); + fail_unless(pjob->ji_qs.ji_state == JOB_STATE_HELD); + } +END_TEST + + START_TEST(test_job_ids_match) { is_attr_set = 1; @@ -848,6 +869,7 @@ Suite *req_register_suite(void) tc_core = tcase_create("cat_jobsvr_test"); tcase_add_test(tc_core, cat_jobsvr_test); tcase_add_test(tc_core, fast_strcat_test); + tcase_add_test(tc_core, test_depend_on_que); tc_core = tcase_create("set_depend_test"); tcase_add_test(tc_core, set_depend_test); From 42c00b161b56d20f326fc1fe583122f2aca4a7f1 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 2 May 2017 16:01:56 -0600 Subject: [PATCH 248/312] TRQ-3972. Make place=socket|numa work with gpus. Conflicts: src/test/numa_chip/test_uut.c --- src/include/allocation.hpp | 2 + src/include/machine.hpp | 5 ++- src/lib/Libutils/allocation.cpp | 71 ++++++++++++++++++++++++++++++ src/lib/Libutils/machine.cpp | 11 ++--- src/lib/Libutils/numa_chip.cpp | 65 +++++++++++++++------------ src/lib/Libutils/numa_socket.cpp | 39 +++++++--------- src/test/allocation/test_uut.c | 26 +++++++++++ src/test/machine/scaffolding.c | 6 ++- src/test/numa_chip/scaffolding.c | 1 + src/test/numa_chip/test_uut.c | 42 +++++++++++------- src/test/numa_socket/scaffolding.c | 10 ++++- src/test/numa_socket/test_uut.c | 14 +++--- 12 files changed, 209 insertions(+), 83 deletions(-) diff --git a/src/include/allocation.hpp b/src/include/allocation.hpp index 09be6ff473..a63e67f14b 100644 --- a/src/include/allocation.hpp +++ b/src/include/allocation.hpp @@ -165,6 +165,8 @@ class allocation bool fully_placed() const; bool partially_placed(const req &r) const; void clear(); + void adjust_for_spread(unsigned int quantity, bool use_modulo); + void adjust_for_remainder(allocation &remainder); }; #endif diff --git a/src/include/machine.hpp b/src/include/machine.hpp index ddb2be8592..dfae0010f2 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -188,12 +188,13 @@ class Chip void setCores(int cores); // used for unit tests void setThreads(int threads); // used for unit tests void setChipAvailable(bool available); + void set_gpus(int gpus); // used for unit tests double how_many_tasks_fit(const req &r, int place_type) const; bool has_socket_exclusive_allocation() const; bool task_will_fit(const req &r, int place_type) const; int free_core_count() const; void calculateStepCounts(const int lprocs_per_task, const int pu_per_task, int &step, int &step_rem, int &place_count, int &place_count_rem); - bool spread_place(req &r, allocation &master, int execution_slots_per, int &remainder); + bool spread_place(req &r, allocation &master, allocation &to_place, allocation &remainder); bool spread_place_cores(req &r, allocation &master, int &remaining_cores, int &remaining_lprocs, int &gpus, int &mics); bool spread_place_threads(req &r, allocation &master, int &remaining_cores, int &remaining_lprocs, int &gpus, int &mics); void place_all_execution_slots(req &r, allocation &task_alloc); @@ -283,7 +284,7 @@ class Socket void addChip(); // used for unit tests double how_many_tasks_fit(const req &r, int place_type) const; void place_all_execution_slots(req &r, allocation &task_alloc); - bool spread_place(req &r, allocation &master, int execution_slots_per, int &remainder, bool chips); + bool spread_place(req &r, allocation &master, allocation &to_place, allocation &remainder, bool chips); bool spread_place_pu(req &r, allocation &task_alloc, int &cores, int &lprocs, int &gpus, int &mics); int place_task(req &r, allocation &a, int to_place, const char *hostname); bool free_task(const char *jobid); diff --git a/src/lib/Libutils/allocation.cpp b/src/lib/Libutils/allocation.cpp index c8362b6498..c0f2a9a9f2 100644 --- a/src/lib/Libutils/allocation.cpp +++ b/src/lib/Libutils/allocation.cpp @@ -610,3 +610,74 @@ bool allocation::partially_placed( ((r.getPlaceThreads() > 0) && (this->place_cpus != r.getPlaceThreads()))); } + + + +/* + * adjust_for_spread() + * + * Reduces the amount of resources to place for this allocation according to how + * many things it is being spread across. + * + * @param quantity - the number of things it is being spread across + * @param finding_remainder - tells us whether or not we're dividing or finding a remainder + */ + +void allocation::adjust_for_spread( + + unsigned int quantity, + bool finding_remainder) + + { + if (quantity != 0) + { + if (finding_remainder == false) + { + this->cpus /= quantity; + this->gpus /= quantity; + this->mics /= quantity; + } + else + { + this->cpus %= quantity; + this->gpus %= quantity; + this->mics %= quantity; + } + } + } // END adjust_for_spread() + + + +/* + * adjust_for_remainder() + * + * Optionally increases the amount of resources to place for this allocation according to the + * remainder parameter. Also, reduces remainder to account for the adjustment. + * + * @param remainder - the remainder that needs to be accounted for. + */ + +void allocation::adjust_for_remainder( + + allocation &remainder) + + { + if (remainder.cpus > 0) + { + this->cpus++; + remainder.cpus--; + } + + if (remainder.mics > 0) + { + this->mics++; + remainder.mics--; + } + + if (remainder.gpus > 0) + { + this->gpus++; + remainder.gpus--; + } + } // END adjust_for_remainder() + diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index 75539fc566..0b7f86d0eb 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -1062,22 +1062,23 @@ int Machine::spread_place( } // Spread the placement evenly across the number of sockets or numa nodes - int execution_slots_per = r.getExecutionSlots() / quantity; - int execution_slots_remainder = r.getExecutionSlots() % quantity; - for (int i = 0; i < tasks_for_node; i++) { bool partial_place = false; allocation task_alloc(master.jobid.c_str()); task_alloc.set_place_type(r.getPlacementType()); + allocation remaining(r); + allocation remainder(r); + + remaining.adjust_for_spread(quantity, false); + remainder.adjust_for_spread(quantity, true); for (int j = 0; j < quantity; j++) { for (unsigned int s = 0; s < this->sockets.size(); s++) { - if (this->sockets[s].spread_place(r, task_alloc, execution_slots_per, - execution_slots_remainder, chips)) + if (this->sockets[s].spread_place(r, task_alloc, remaining, remainder, chips)) { partial_place = true; diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index ede678836b..84ccffbad9 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -1049,6 +1049,21 @@ void Chip::setCores( this->availableCores = cores; } +void Chip::set_gpus( + + int gpus) + + { + for (int i = 0; i < gpus; i++) + { + PCI_Device p; + p.set_type(GPU); + p.setId(i); + this->total_gpus++; + this->devices.push_back(p); + } + } + void Chip::setThreads( int threads) @@ -2117,40 +2132,37 @@ bool Chip::spread_place( req &r, allocation &task_alloc, - int execution_slots_per, - int &execution_slots_remainder) + allocation &remaining, + allocation &remainder) { bool task_placed = false; if ((this->is_completely_free() == true) && - ((execution_slots_per + execution_slots_remainder) <= this->totalThreads)) + ((remaining.cpus + remainder.cpus) <= this->totalThreads)) { allocation from_this_chip(task_alloc.jobid.c_str()); int placed = 0; - int to_add = 0; + int execution_slots = remaining.cpus; - if (execution_slots_remainder > 0) - to_add = 1; + if (remainder.cpus > 0) + { + execution_slots++; + remainder.cpus--; + } from_this_chip.place_type = task_alloc.place_type; - if (this->totalCores >= execution_slots_per + to_add) + if (this->totalCores >= execution_slots) { - if (to_add == 1) - { - execution_slots_per++; - execution_slots_remainder--; - } - // Spread over just the cores float step = 1.0; - if (execution_slots_per > 0) - step = this->totalCores / (float)execution_slots_per; + if (execution_slots > 0) + step = this->totalCores / (float)execution_slots; from_this_chip.cores_only = true; - for (float i = 0.0; placed < execution_slots_per; i+= step) + for (float i = 0.0; placed < execution_slots; i+= step) { this->reserve_core(std::floor(i + 0.5), from_this_chip); placed++; @@ -2160,17 +2172,10 @@ bool Chip::spread_place( { // Spread over the cores and the threads - this option doesn't really make any // sense but I suppose we have to code for it - int per_core = execution_slots_per / this->totalCores; - int remainder = execution_slots_per % this->totalCores; + int per_core = execution_slots / this->totalCores; + int leftover = execution_slots % this->totalCores; - // Place one extra if we still have a remainder - if (execution_slots_remainder > 0) - { - placed--; - execution_slots_remainder--; - } - - for (unsigned int i = 0; i < this->cores.size() && placed < execution_slots_per; i++) + for (unsigned int i = 0; i < this->cores.size() && placed < execution_slots; i++) { for (int j = 0; j < per_core; j++) { @@ -2179,14 +2184,14 @@ bool Chip::spread_place( } } - while (remainder > 0) + while (leftover > 0) { - for (unsigned int i = 0; i < this->cores.size() && placed < execution_slots_per; i++) + for (unsigned int i = 0; i < this->cores.size() && placed < execution_slots; i++) { if (this->reserve_thread(i, from_this_chip) == true) { placed++; - remainder--; + leftover--; } } } @@ -2194,6 +2199,8 @@ bool Chip::spread_place( from_this_chip.mem_indices.push_back(this->id); + place_accelerators(remaining, from_this_chip); + place_accelerators(remainder, from_this_chip); task_placed = true; this->chip_exclusive = true; this->aggregate_allocation(from_this_chip); diff --git a/src/lib/Libutils/numa_socket.cpp b/src/lib/Libutils/numa_socket.cpp index 4faf4ad8a2..22279b7420 100644 --- a/src/lib/Libutils/numa_socket.cpp +++ b/src/lib/Libutils/numa_socket.cpp @@ -636,15 +636,13 @@ bool Socket::spread_place( req &r, allocation &task_alloc, - int execution_slots_per, - int &execution_slots_remainder, + allocation &remaining, + allocation &remainder, bool chip) { bool placed = false; int numa_nodes_required = 1; - int per_numa = execution_slots_per; - int per_numa_remainder = 0; // We must either be completely free or be placing on just one chip if ((this->is_completely_free()) || @@ -654,33 +652,30 @@ bool Socket::spread_place( { // If we're placing at the socket level, divide execution_slots_per by the number // of chips and place multiple times - per_numa_remainder = per_numa % this->chips.size(); - per_numa /= this->chips.size(); numa_nodes_required = this->chips.size(); - this->socket_exclusive = true; } for (int c = 0; c < numa_nodes_required; c++) { + allocation per_numa(remaining); + allocation numa_remainder(remaining); + + if (numa_nodes_required == 1) + numa_remainder.clear(); + else + { + per_numa.adjust_for_spread(numa_nodes_required, false); + numa_remainder.adjust_for_spread(numa_nodes_required, true); + } + for (unsigned int i = 0; i < this->chips.size(); i++) { - if (per_numa_remainder > 0) - { - if (this->chips[i].spread_place(r, task_alloc, per_numa + 1, execution_slots_remainder)) - { - placed = true; - per_numa_remainder--; - break; - } - } - else + per_numa.adjust_for_remainder(numa_remainder); + if (this->chips[i].spread_place(r, task_alloc, per_numa, remainder)) { - if (this->chips[i].spread_place(r, task_alloc, per_numa, execution_slots_remainder)) - { - placed = true; - break; - } + placed = true; + break; } } } diff --git a/src/test/allocation/test_uut.c b/src/test/allocation/test_uut.c index cab32f7560..3a71283a0e 100644 --- a/src/test/allocation/test_uut.c +++ b/src/test/allocation/test_uut.c @@ -9,6 +9,31 @@ extern int tc; +START_TEST(test_remainder_things) + { + allocation a; + + a.cpus = 9; + a.gpus = 3; + allocation b(a); + + a.adjust_for_spread(2, false); + b.adjust_for_spread(2, true); + + fail_unless(a.cpus == 4); + fail_unless(a.gpus == 1); + fail_unless(b.cpus == 1); + fail_unless(b.gpus == 1); + + a.adjust_for_remainder(b); + fail_unless(b.cpus == 0); + fail_unless(b.gpus == 0); + fail_unless(a.cpus == 5); + fail_unless(a.gpus == 2); + } +END_TEST + + START_TEST(test_write_task_information) { allocation a; @@ -254,6 +279,7 @@ Suite *allocation_suite(void) tcase_add_test(tc_core, test_allocation_constructors); tcase_add_test(tc_core, test_set_place_type); tcase_add_test(tc_core, test_fully_placed); + tcase_add_test(tc_core, test_remainder_things); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_add_allocation"); diff --git a/src/test/machine/scaffolding.c b/src/test/machine/scaffolding.c index 76cf63448a..59a07f3a32 100644 --- a/src/test/machine/scaffolding.c +++ b/src/test/machine/scaffolding.c @@ -151,8 +151,8 @@ bool Socket::spread_place( req &r, allocation &master, - int execution_slots_per, - int &execution_slots_remainder, + allocation &remaining, + allocation &remainder, bool chips) { @@ -385,6 +385,8 @@ allocation &allocation::operator =(const allocation &other) return(*this); } +void allocation::adjust_for_spread(unsigned int quantity, bool find_remainder) {} + PCI_Device::~PCI_Device() {} PCI_Device::PCI_Device() {} PCI_Device::PCI_Device(const PCI_Device &other) {} diff --git a/src/test/numa_chip/scaffolding.c b/src/test/numa_chip/scaffolding.c index 8c6b02d89d..e621557af8 100644 --- a/src/test/numa_chip/scaffolding.c +++ b/src/test/numa_chip/scaffolding.c @@ -278,6 +278,7 @@ void setAllocJson(int test_num) alloc_json[ALLOCATIONS][0][ALLOCATION][CPUS] = "0,4,8,12"; alloc_json[ALLOCATIONS][0][ALLOCATION][EXCLUSIVE] = 3; alloc_json[ALLOCATIONS][0][ALLOCATION][CORES_ONLY] = 1; + alloc_json[GPUS] = "0-1"; return; } }//end test_num <5 diff --git a/src/test/numa_chip/test_uut.c b/src/test/numa_chip/test_uut.c index 41c38db5b8..c33641d3ae 100644 --- a/src/test/numa_chip/test_uut.c +++ b/src/test/numa_chip/test_uut.c @@ -420,22 +420,23 @@ START_TEST(test_spread_place) Json::Value out; allocation a(jobid); + allocation remaining; + allocation remainder; Chip c; c.setId(0); c.setThreads(32); c.setCores(16); c.setMemory(6); c.setChipAvailable(true); + c.set_gpus(2); for (int i = 0; i < 16; i++) c.make_core(i); - int remaining = 0; a.place_type = exclusive_chip; // Make sure we get 4 evenly spread cores - fail_unless(c.spread_place(r, a, 4, remaining) == true); - - + remaining.cpus = 4; + fail_unless(c.spread_place(r, a, remaining, remainder) == true); setAllocJson(4);//initialize alloc_json to this tests configuration c.displayAsJson(out, true); @@ -446,14 +447,19 @@ START_TEST(test_spread_place) // Make sure we do not place another task on this chip before freeing the old one fail_unless(c.reserve_core(0, a) == false); - remaining = 1; - fail_unless(c.spread_place(r, a, 4, remaining) == false); + remaining.cpus = 4; + remainder.cpus = 1; + remaining.gpus = 1; + remainder.gpus = 1; + fail_unless(c.spread_place(r, a, remaining, remainder) == false); fail_unless(c.has_socket_exclusive_allocation() == false); c.free_task(jobid); // Check that we place 5 cores correctly - fail_unless(c.spread_place(r, a, 4, remaining) == true); - fail_unless(remaining == 0, "remaining = %d", remaining); + fail_unless(c.spread_place(r, a, remaining, remainder) == true); + fail_unless(remainder.cpus == 0, "remainder = %d", remainder.cpus); + fail_unless(remainder.gpus == 0, "remainder = %d", remainder.gpus); + fail_unless(remaining.gpus == 0, "remainder = %d", remaining.gpus); fail_unless(c.reserve_core(0, a) == false); fail_unless(c.reserve_core(3, a) == false); fail_unless(c.reserve_core(6, a) == false); @@ -462,15 +468,17 @@ START_TEST(test_spread_place) c.free_task(jobid); // Make sure we get core 0 if we request 0 + 1 in the remainder - remaining = 1; - fail_unless(c.spread_place(r, a, 0, remaining) == true); + remaining.cpus = 0; + remainder.cpus = 1; + fail_unless(c.spread_place(r, a, remaining, remainder) == true); fail_unless(c.reserve_core(0, a) == false); - fail_unless(remaining == 0); + fail_unless(remainder.cpus == 0); // Check what happens we we place an empty set on the chip c.free_task(jobid); a.place_type = exclusive_socket; - fail_unless(c.spread_place(r, a, 0, remaining) == true); + remaining.cpus = 0; + fail_unless(c.spread_place(r, a, remaining, remainder) == true); fail_unless(c.getAvailableCores() == 0); fail_unless(c.getAvailableThreads() == 0); fail_unless(c.has_socket_exclusive_allocation() == true); @@ -478,9 +486,10 @@ START_TEST(test_spread_place) c.free_task(jobid); allocation a2(jobid2); a2.place_type = exclusive_chip; - remaining = 0; - fail_unless(c.spread_place(r, a2, 40, remaining) == false); - fail_unless(c.spread_place(r, a2, 18, remaining) == true); + remaining.cpus = 40; + fail_unless(c.spread_place(r, a2, remaining, remainder) == false); + remaining.cpus = 18; + fail_unless(c.spread_place(r, a2, remaining, remainder) == true); fail_unless(c.getAvailableCores() == 0); fail_unless(c.getAvailableThreads() == 0); c.free_task(jobid2); @@ -493,7 +502,8 @@ START_TEST(test_spread_place) int tasks = c.place_task(r2, a3, 1, host); fail_unless(tasks == 1); // We shouldn't place anything in spread place unless we're completely free - fail_unless(c.spread_place(r, a2, 18, remaining) == false); + remaining.cpus = 18; + fail_unless(c.spread_place(r, a2, remaining, remainder) == false); } END_TEST diff --git a/src/test/numa_socket/scaffolding.c b/src/test/numa_socket/scaffolding.c index bafc901bea..625578a163 100644 --- a/src/test/numa_socket/scaffolding.c +++ b/src/test/numa_socket/scaffolding.c @@ -152,8 +152,8 @@ bool Chip::spread_place( req &r, allocation &master, - int execution_slots_per, - int &remaining) + allocation &remaining, + allocation &remainder) { called_spread_place++; @@ -366,6 +366,12 @@ void allocation::set_place_type( else this->place_type = exclusive_none; } // END set_place_type() + +void allocation::adjust_for_spread(unsigned int quantity, bool find_remainder) {} + +void allocation::adjust_for_remainder(allocation &remainder) {} + +void allocation::clear() {} bool Chip::spread_place_threads( diff --git a/src/test/numa_socket/test_uut.c b/src/test/numa_socket/test_uut.c index 4d43332948..0c7be6ebf3 100644 --- a/src/test/numa_socket/test_uut.c +++ b/src/test/numa_socket/test_uut.c @@ -66,25 +66,29 @@ START_TEST(test_spread_place) Socket s; req r; allocation a; - int remaining = 0; + allocation remaining; + allocation remainder; s.addChip(); s.addChip(); completely_free = true; called_spread_place = 0; - fail_unless(s.spread_place(r, a, 5, remaining, false) == true); + remaining.cpus = 5; + fail_unless(s.spread_place(r, a, remaining, remainder, false) == true); fail_unless(called_spread_place == 2); completely_free = false; - fail_unless(s.spread_place(r, a, 5, remaining, false) == false); + remaining.cpus = 5; + fail_unless(s.spread_place(r, a, remaining, remainder, false) == false); fail_unless(called_spread_place == 2); completely_free = true; oscillate = false; s.free_task("1.napali"); - fail_unless(s.spread_place(r, a, 5, remaining, true) == true); + remaining.cpus = 5; + fail_unless(s.spread_place(r, a, remaining, remainder, true) == true); fail_unless(called_spread_place == 3); - fail_unless(s.spread_place(r, a, 5, remaining, true) == true); + fail_unless(s.spread_place(r, a, remaining, remainder, true) == true); fail_unless(called_spread_place == 4); } From 6dd0fe8887e3c4bd8cbadc43541e77e284525e00 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 4 May 2017 00:33:58 -0600 Subject: [PATCH 249/312] TRQ-3634 - display correct output paths when qsub -k used --- src/resmom/start_exec.c | 71 ++++++++++++++++++++++++++++++++++ src/test/start_exec/test_uut.c | 18 +++++++++ 2 files changed, 89 insertions(+) diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index 0be6cfac72..ef2e5dca9f 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -381,6 +381,7 @@ extern int use_nvidia_gpu; #endif /* NVIDIA_GPUS */ int exec_job_on_ms(job *pjob); +int remove_leading_hostname(char**); @@ -2157,7 +2158,71 @@ struct radix_buf **allocate_sister_list( } /* END allocate_sister_list() */ +/* + * update the job outpath/errpath attribute with this hostname if needed + * + * @param pjob - the job to check + * @param attr_index - must be JOB_ATR_outpath or JOB_ATR_errpath + * @return -1 on failure + */ + +int update_path_attribute( + + job *pjob, + job_atr attr_index) + + { + char outchar; + + if (pjob == NULL) + return(-1); + + if (attr_index == JOB_ATR_outpath) + outchar = 'o'; + else if (attr_index == JOB_ATR_errpath) + outchar = 'e'; + else + return(-1); + + // if keeping output on this host, update the hostname + if ((pjob->ji_wattr[JOB_ATR_keep].at_flags & ATR_VFLAG_SET) && + (strchr(pjob->ji_wattr[JOB_ATR_keep].at_val.at_str, outchar)) && + (spoolasfinalname == FALSE)) + { + char *p; + string full_path; + pbs_attribute *pattr; + // get the current path (may include leading hostname) + p = pjob->ji_wattr[attr_index].at_val.at_str; + + // get just the file path + remove_leading_hostname(&p); + + // build new host:path string + full_path = mom_host; + full_path += ":"; + full_path += p; + + // update the attribute + + pattr = &pjob->ji_wattr[attr_index]; + + job_attr_def[attr_index].at_free(pattr); + + job_attr_def[attr_index].at_decode( + pattr, + NULL, + NULL, + full_path.c_str(), + 0); + + pjob->ji_wattr[attr_index].at_flags = + (ATR_VFLAG_SET | ATR_VFLAG_MODIFY | ATR_VFLAG_SEND); + } + + return(0); + } /* * Used by MOM superior to start the shell process. @@ -2484,6 +2549,12 @@ int TMomFinalizeJob1( pjob->ji_wattr[JOB_ATR_errpath].at_flags = (ATR_VFLAG_SET | ATR_VFLAG_MODIFY | ATR_VFLAG_SEND); } /* END if (TJE->is_interactive == TRUE) */ + else + { + // update the hostname in the out/err path attributes if needed + update_path_attribute(pjob, JOB_ATR_outpath); + update_path_attribute(pjob, JOB_ATR_errpath); + } #if SHELL_USE_ARGV == 0 #if SHELL_INVOKE == 1 diff --git a/src/test/start_exec/test_uut.c b/src/test/start_exec/test_uut.c index 203b948d68..a6b3324f21 100644 --- a/src/test/start_exec/test_uut.c +++ b/src/test/start_exec/test_uut.c @@ -23,6 +23,7 @@ int get_indices_from_exec_str(const char *exec_str, char *buf, int buf_size); int remove_leading_hostname(char **jobpath); int get_num_nodes_ppn(const char*, int*, int*); int setup_process_launch_pipes(int &kid_read, int &kid_write, int &parent_read, int &parent_write); +int update_path_attribute(job*, job_atr); #ifdef NUMA_SUPPORT extern nodeboard node_boards[]; @@ -35,6 +36,7 @@ extern char mom_alias[]; char *penv[MAX_TEST_ENVP]; /* max number of pointers bld_env_variables will create */ char *envBuffer = NULL; /* points to the max block that bld_env_variables would ever need in this test suite */ +int spoolasfinalname = FALSE; extern int logged_event; extern int num_contacted; extern int send_sisters_called; @@ -782,6 +784,18 @@ START_TEST(test_TMomCheckJobChild) } END_TEST +START_TEST(test_update_path_attribute) + { + job *pjob = NULL; + + fail_unless(update_path_attribute(pjob, JOB_ATR_outpath) != 0); + pjob = (job *)calloc(1, sizeof(job)); + fail_unless(update_path_attribute(pjob, JOB_ATR_LAST) != 0); + fail_unless(update_path_attribute(pjob, JOB_ATR_outpath) == 0); + fail_unless(update_path_attribute(pjob, JOB_ATR_errpath) == 0); + } +END_TEST + Suite *start_exec_suite(void) { Suite *s = suite_create("start_exec_suite methods"); @@ -851,6 +865,10 @@ Suite *start_exec_suite(void) tcase_add_test(tc_core, test_TMomCheckJobChild); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_update_path_attribute"); + tcase_add_test(tc_core, test_update_path_attribute); + suite_add_tcase(s, tc_core); + return s; } From 1e5c715d540f994ac1fb87d7d337d5ccc5f9b74a Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 4 May 2017 08:41:03 -0600 Subject: [PATCH 250/312] TRQ-3634 - fix build issue under CentOS7 --- src/resmom/start_exec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index ef2e5dca9f..a286594831 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -2190,7 +2190,7 @@ int update_path_attribute( (spoolasfinalname == FALSE)) { char *p; - string full_path; + std::string full_path; pbs_attribute *pattr; // get the current path (may include leading hostname) From 00f97df4e938f3946cc948443ca3d9c8841a7e33 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 4 May 2017 11:30:07 -0600 Subject: [PATCH 251/312] TRQ-4027. Fix a valgrind leak. Conflicts: src/daemon_client/trq_auth_daemon.c --- src/daemon_client/trq_auth_daemon.c | 14 ++--- src/include/lib_ifl.h | 2 +- src/lib/Libifl/trq_auth.c | 88 +++++++++++++++-------------- src/test/trq_auth/test_uut.c | 10 ++-- 4 files changed, 60 insertions(+), 54 deletions(-) diff --git a/src/daemon_client/trq_auth_daemon.c b/src/daemon_client/trq_auth_daemon.c index 57a8a17d7d..f6e2ff808c 100644 --- a/src/daemon_client/trq_auth_daemon.c +++ b/src/daemon_client/trq_auth_daemon.c @@ -43,15 +43,15 @@ bool down_server = false; bool use_log = true; bool daemonize_server = true; static int changed_msg_daem = 0; -static char *active_pbs_server; +std::string active_pbs_server; std::string log_file_path = ""; /* Get the name of the active pbs_server */ int load_trqauthd_config( - char **default_server_name, - int *t_port, - char **trqauthd_unix_domain_port) + std::string &default_server_name, + int *t_port, + char **trqauthd_unix_domain_port) { int rc = PBSE_NONE; @@ -70,7 +70,7 @@ int load_trqauthd_config( * the client utilities determine the pbs_server port) */ printf("hostname: %s\n", tmp_name); - *default_server_name = tmp_name; + default_server_name = tmp_name; PBS_get_server(tmp_name, (unsigned int *)t_port); if (*t_port == 0) *t_port = PBS_BATCH_SERVICE_PORT; @@ -107,7 +107,7 @@ void initialize_globals_for_log(const char *port) strcpy(pbs_current_user, "trqauthd"); if ((msg_daemonname = strdup(pbs_current_user))) changed_msg_daem = 1; - log_set_hostname_sharelogging(active_pbs_server, port); + log_set_hostname_sharelogging(active_pbs_server.c_str(), port); } int init_trqauth_log(const char *server_port) @@ -441,7 +441,7 @@ int trq_main( return(rc); } - if ((rc = load_trqauthd_config(&active_pbs_server, &trq_server_port, &daemon_port)) != PBSE_NONE) + if ((rc = load_trqauthd_config(active_pbs_server, &trq_server_port, &daemon_port)) != PBSE_NONE) { fprintf(stderr, "Failed to load configuration. Make sure the $TORQUE_HOME/server_name file exists\n"); } diff --git a/src/include/lib_ifl.h b/src/include/lib_ifl.h index 007487bcd3..ad933faccf 100644 --- a/src/include/lib_ifl.h +++ b/src/include/lib_ifl.h @@ -27,7 +27,7 @@ int parse_response_svr(int sock, char **msg); int build_response_client(int code, char *msg, char **send_message); int get_trq_server_addr(char *server_name, char **server_addr, int *server_addr_len); void *process_svr_conn(void *sock); -int validate_server(char *active_server_name, int t_server_port, char *ssh_key, char **sign_key); +int validate_server(std::string &active_server_name, int t_server_port, char *ssh_key, char **sign_key); int set_active_pbs_server(const char *, const int); int get_active_pbs_server(char **, int *); int validate_active_pbs_server(char **); diff --git a/src/lib/Libifl/trq_auth.c b/src/lib/Libifl/trq_auth.c index a50ebf4a83..5e0fced2c5 100644 --- a/src/lib/Libifl/trq_auth.c +++ b/src/lib/Libifl/trq_auth.c @@ -70,7 +70,7 @@ int set_active_pbs_server( ) { - strncpy(active_pbs_server, new_active_server, PBS_MAXSERVERNAME); + snprintf(active_pbs_server, sizeof(active_pbs_server), "%s", new_active_server); active_pbs_server_port = new_active_port; return(PBSE_NONE); } @@ -401,10 +401,10 @@ int trq_simple_connect( * server is made the active server */ int validate_server( - char *active_server_name, - int t_server_port, - char *ssh_key, - char **sign_key) + std::string &active_server_name, + int t_server_port, + char *ssh_key, + char **sign_key) { int rc = PBSE_NONE; @@ -420,8 +420,8 @@ int validate_server( we stick with the active_server_name. If another server in teh server_name_list responds that will become the active_pbs_server */ - if (active_server_name != NULL) - rc = trq_simple_connect(active_server_name, t_server_port, &sd); + if (active_server_name.size() > 0) + rc = trq_simple_connect(active_server_name.c_str(), t_server_port, &sd); else rc = PBSE_UNKREQ; /* If we don't have a server name we want to process everyting. */ @@ -520,19 +520,24 @@ int parse_terminate_request( int parse_request_client( - int sock, - char **server_name, - int *server_port, - int *auth_type, - char **user, - int *user_pid, - int *user_sock) + int sock, + std::string &server_name, + int *server_port, + int *auth_type, + char **user, + int *user_pid, + int *user_sock) { - int rc = PBSE_NONE; - long long tmp_val = 0, tmp_port = 0, tmp_auth_type = 0, tmp_sock = 0, tmp_pid = 0; + int rc = PBSE_NONE; + long long tmp_val = 0; + long long tmp_port = 0; + long long tmp_auth_type = 0; + long long tmp_sock = 0; + long long tmp_pid = 0; + char *ptr = NULL; - if ((rc = socket_read_str(sock, server_name, &tmp_val)) != PBSE_NONE) + if ((rc = socket_read_str(sock, &ptr, &tmp_val)) != PBSE_NONE) { } else if ((rc = socket_read_num(sock, &tmp_port)) != PBSE_NONE) @@ -557,7 +562,14 @@ int parse_request_client( *user_pid = (int)tmp_pid; *user_sock = (int)tmp_sock; } - return rc; + + if (ptr != NULL) + { + server_name = ptr; + free(ptr); + } + + return(rc); } @@ -627,7 +639,8 @@ int build_active_server_response( if (len == 0) { - validate_server(NULL, 0, NULL, NULL); + std::string empty; + validate_server(empty, 0, NULL, NULL); len = strlen(active_pbs_server); } @@ -760,9 +773,9 @@ int parse_response_svr( int get_trq_server_addr( - char *server_name, - char **server_addr, - int *server_addr_len) + const char *server_name, + char **server_addr, + int *server_addr_len) { int rc = PBSE_NONE; @@ -845,7 +858,7 @@ int authorize_socket( int local_socket, std::string &message, char *msg_buf, - char **server_name_ptr, + std::string &server_name, char **user_name_ptr, std::string &err_msg) @@ -877,11 +890,8 @@ int authorize_socket( * 0|0|| */ - if ((rc = parse_request_client(local_socket, server_name_ptr, &server_port, &auth_type, user_name_ptr, &user_pid, &user_sock)) != PBSE_NONE) + if ((rc = parse_request_client(local_socket, server_name, &server_port, &auth_type, user_name_ptr, &user_pid, &user_sock)) != PBSE_NONE) { - if (*server_name_ptr != NULL) - free(*server_name_ptr); - if (*user_name_ptr != NULL) free(*user_name_ptr); @@ -890,7 +900,6 @@ int authorize_socket( else { int retries = 0; - char *server_name = *server_name_ptr; while (retries < MAX_RETRIES) { @@ -905,7 +914,7 @@ int authorize_socket( usleep(20000); continue; } - else if ((rc = get_trq_server_addr(server_name, &trq_server_addr, &trq_server_addr_len)) != PBSE_NONE) + else if ((rc = get_trq_server_addr(server_name.c_str(), &trq_server_addr, &trq_server_addr_len)) != PBSE_NONE) { disconnect_svr = false; retries++; @@ -977,11 +986,11 @@ int authorize_socket( if (debug_mode == TRUE) { fprintf(stderr, "Conn to %s port %d success. Conn %d authorized\n", - server_name, server_port, user_sock); + server_name.c_str(), server_port, user_sock); } sprintf(msg_buf, - "User %s at IP:port %s:%d logged in", *user_name_ptr, server_name, server_port); + "User %s at IP:port %s:%d logged in", *user_name_ptr, server_name.c_str(), server_port); log_record(PBSEVENT_CLIENTAUTH | PBSEVENT_FORCE, PBS_EVENTCLASS_TRQAUTHD, className, msg_buf); } @@ -1010,7 +1019,7 @@ void *process_svr_conn( { const char *className = "trqauthd"; int rc = PBSE_NONE; - char *server_name = NULL; + std::string server_name; int server_port = 0; char *user_name = NULL; int user_pid = 0; @@ -1074,7 +1083,7 @@ void *process_svr_conn( case TRQ_AUTH_CONNECTION: { - rc = authorize_socket(local_socket, message, msg_buf, &server_name, &user_name, error_string); + rc = authorize_socket(local_socket, message, msg_buf, server_name, &user_name, error_string); break; } default: @@ -1113,25 +1122,25 @@ void *process_svr_conn( if (debug_mode == TRUE) { - if (server_name != NULL) + if (server_name.size() > 0) fprintf(stderr, "Conn to %s port %d Fail. Conn %d not authorized (Err Num %d)\n", - server_name, server_port, user_sock, rc); + server_name.c_str(), server_port, user_sock, rc); } if (error_string.size() == 0) { - if (server_name != NULL) + if (server_name.size() > 0) snprintf(msg_buf, sizeof(msg_buf), "User %s at IP:port %s:%d login attempt failed --no message", (user_name) ? user_name : "null", - server_name, server_port); + server_name.c_str(), server_port); } else { snprintf(msg_buf, sizeof(msg_buf), "User %s at IP:port %s:%d login attempt failed --%s", (user_name) ? user_name : "null", - (server_name) ? server_name : "null", server_port, + (server_name.size()) ? server_name.c_str() : "null", server_port, error_string.c_str()); } log_record(PBSEVENT_CLIENTAUTH | PBSEVENT_FORCE, PBS_EVENTCLASS_TRQAUTHD, @@ -1141,9 +1150,6 @@ void *process_svr_conn( if (message.length() != 0) rc = socket_write(local_socket, message.c_str(), message.length()); - if (server_name != NULL) - free(server_name); - if (user_name != NULL) free(user_name); diff --git a/src/test/trq_auth/test_uut.c b/src/test/trq_auth/test_uut.c index 5bfac6f8bd..37cf40a31f 100644 --- a/src/test/trq_auth/test_uut.c +++ b/src/test/trq_auth/test_uut.c @@ -159,9 +159,10 @@ START_TEST(test_trq_simple_disconnect) } END_TEST + START_TEST(test_validate_server) { - char active_server_name[PBS_MAXHOSTNAME+1]; + std::string active_server("localhost"); char *ssh_key = NULL; char *sign_key = NULL; int rc; @@ -180,13 +181,12 @@ START_TEST(test_validate_server) socket_connect_success = true; DIS_success = true; - strcpy(active_server_name, "localhost"); - rc = validate_server(active_server_name, port, ssh_key, &sign_key); + rc = validate_server(active_server, port, ssh_key, &sign_key); fail_unless(rc == PBSE_NONE, "validate_server success case failed", rc); - } END_TEST + START_TEST(test_set_active_pbs_server) { char new_server_name[PBS_MAXHOSTNAME + 1]; @@ -195,10 +195,10 @@ START_TEST(test_set_active_pbs_server) strcpy(new_server_name, "localhost"); rc = set_active_pbs_server(new_server_name, 15001); fail_unless(rc == PBSE_NONE, "set_active_pbs_server failed", rc); - } END_TEST + START_TEST(test_validate_active_pbs_server) { int rc; From 28dbe04ceaaadd1c88aa5e581e2b32aaf846a943 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 4 May 2017 14:16:42 -0600 Subject: [PATCH 252/312] TRQ-3951. Make the removeAfterAnyDependency() function delete empty dependencies. --- src/server/req_register.c | 6 ++++++ src/test/req_register/test_uut.c | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/server/req_register.c b/src/server/req_register.c index dd217b7ad6..f97a4e37b0 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -3357,6 +3357,12 @@ void removeAfterAnyDependency( if (pDepJob != NULL) { del_depend_job(pDep, pDepJob); + + if (pDep->dp_jobs.size() == 0) + { + /* no more dependencies of this type */ + delete pDep; + } try { diff --git a/src/test/req_register/test_uut.c b/src/test/req_register/test_uut.c index c9f89c9675..762f8af280 100644 --- a/src/test/req_register/test_uut.c +++ b/src/test/req_register/test_uut.c @@ -834,10 +834,7 @@ START_TEST(remove_after_any_test) pattr = &pJob->ji_wattr[JOB_ATR_depend]; pdep = find_depend(JOB_DEPEND_TYPE_AFTERANY,pattr); - pj = find_dependjob(pdep,pTJob->ji_qs.ji_jobid); - fail_unless((pj == NULL),"Dependency not deleted."); - pj = find_dependjob(pdep,pOJob->ji_qs.ji_jobid); - fail_unless((pj == NULL),"Dependency not deleted."); + fail_unless(pdep == NULL, "Dependency not deleted"); } END_TEST From 569de629ef33cb0072bdf34972af1a927c54920f Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Thu, 4 May 2017 14:23:39 -0600 Subject: [PATCH 253/312] TRQ-3988 Update distributed spec file to define _initrddir as /etc/init.d if on a system using SYSV init.d --- buildutils/torque.spec.in | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/buildutils/torque.spec.in b/buildutils/torque.spec.in index 79f0b7d40e..6f24df39c0 100644 --- a/buildutils/torque.spec.in +++ b/buildutils/torque.spec.in @@ -13,6 +13,13 @@ %define bcond_without() %{expand:%%{!?_without_%{1}:%%global with_%{1} 1}} %endif +# If _initrddir is defined, which indicates that we are on a system with SYSV +# init.d services, define _initrddir to be /etc/init.d so that the source +# tarball will work across both Red Hat and SUSE distros. +%if %{?_initrddir:1}0 +%define _initrddir /etc/init.d +%endif + # some binaries under /bin in el5|6 are only provided in /usr/bin in el7+. # the el7 rpms actually provide the old location, but can't be used as a rpm requirement # introducing the rootbin macro for this, as none seems to exists From 5a08dccd945871c2e52f56d520b97a86adb0fe22 Mon Sep 17 00:00:00 2001 From: Matt Mix Date: Mon, 8 May 2017 09:23:18 -0500 Subject: [PATCH 254/312] Reset fromlen before each accept() call Accept() will only write fromlen bytes to the buffer it is given, but will then set fromlen to the number of bytes it wanted to write. This causes subsequent calls to write past the from buffer onto stack variables, noticeably the n loop counter which causes a segfault. Fix resets fromlen before each accept() call. --- src/lib/Libnet/port_forwarding.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/Libnet/port_forwarding.c b/src/lib/Libnet/port_forwarding.c index 1a144df346..bed04651b6 100644 --- a/src/lib/Libnet/port_forwarding.c +++ b/src/lib/Libnet/port_forwarding.c @@ -46,7 +46,7 @@ void port_forwarder( torque_socklen_t fromlen; int n, n2, sock; - fromlen = sizeof(from); + while (1) { @@ -102,7 +102,8 @@ void port_forwarder( if ((socks + n)->listening) { int newsock = 0, peersock = 0; - + fromlen = sizeof(from); + if ((sock = accept((socks + n)->sock, (struct sockaddr *) & from, &fromlen)) < 0) { if ((errno == EAGAIN) || (errno == EWOULDBLOCK) || (errno == EINTR) || (errno == ECONNABORTED)) From fa280f1a56ead4db3ee1aabcf776219a258998d8 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 10 May 2017 10:48:39 -0600 Subject: [PATCH 255/312] TRQ-4003. Fix a client-side segfault when using Torque's drmaa library. Also fix a compilation error if you're linking with C and not C++. --- src/drmaa/src/session.c | 3 ++- src/include/pbs_ifl.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/drmaa/src/session.c b/src/drmaa/src/session.c index 86d0a931bb..f5deeddcae 100644 --- a/src/drmaa/src/session.c +++ b/src/drmaa/src/session.c @@ -553,7 +553,8 @@ int drmaa_job_ps( static struct attropl attribs[2]; - if (strcmp(attribs[0].name, "job_state")) + if ((attribs[0].name == NULL) || + (strcmp(attribs[0].name, "job_state"))) init_attribs(attribs); #if 0 { NULL, "exit_status", NULL, NULL, 0 } diff --git a/src/include/pbs_ifl.h b/src/include/pbs_ifl.h index a345b7266d..c2c86d9fed 100644 --- a/src/include/pbs_ifl.h +++ b/src/include/pbs_ifl.h @@ -657,7 +657,9 @@ int pbs_asyrunjob(int c, char *jobid, char *location, char *extend); int pbs_alterjob_async(int connect, char *job_id, struct attrl *attrib, char *extend); int pbs_alterjob(int connect, char *job_id, struct attrl *attrib, char *extend); int pbs_connect(char *server); +#ifdef __cplusplus int pbs_connect_ext(char *server, bool silence); +#endif int pbs_query_max_connections(); char *pbs_default(void); char *pbs_fbserver(void); From 400f1d02245bb8d7f4c99aeb6b2e37479aaa9a92 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 10 May 2017 15:02:58 -0600 Subject: [PATCH 256/312] TRQ-4035. Memory-only nodes were causing the cgroup code to crash. --- src/lib/Libutils/numa_chip.cpp | 7 +++++-- src/test/numa_chip/test_uut.c | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index 84ccffbad9..2ba3f5c1d9 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -229,12 +229,15 @@ void Chip::initialize_cores_from_strings( { std::vector core_indices; std::vector thread_indices; - int ratio; + int ratio = 0; translate_range_string_to_vector(cores_str.c_str(), core_indices); translate_range_string_to_vector(threads_str.c_str(), thread_indices); - ratio = thread_indices.size() / core_indices.size(); + // Check if this is a memory-only node + if (core_indices.size() > 0) + ratio = thread_indices.size() / core_indices.size(); + unsigned int j = 0; for (unsigned int i = 0; i < core_indices.size(); i++) diff --git a/src/test/numa_chip/test_uut.c b/src/test/numa_chip/test_uut.c index c33641d3ae..eb9bbdd729 100644 --- a/src/test/numa_chip/test_uut.c +++ b/src/test/numa_chip/test_uut.c @@ -16,6 +16,21 @@ extern std::string thread_type; extern Json::Value alloc_json; extern void setAllocJson(int); + +START_TEST(test_initialize_cores_from_strings) + { + Chip c; + std::string cores; + std::string threads; + + // Make sure this doesn't segfault + c.initialize_cores_from_strings(cores, threads); + fail_unless(c.getTotalCores() == 0); + fail_unless(c.getTotalThreads() == 0); + } +END_TEST + + START_TEST(test_place_tasks_execution_slots) { const char *jobid = "1.napali"; @@ -1233,6 +1248,7 @@ Suite *numa_socket_suite(void) tcase_add_test(tc_core, test_spread_place_threads); tcase_add_test(tc_core, test_spread_place_cores); tcase_add_test(tc_core, test_spread_place); + tcase_add_test(tc_core, test_initialize_cores_from_strings); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_displayAsString"); From 97a6271a5f52da3910fffcae456e6f315504c221 Mon Sep 17 00:00:00 2001 From: David Beer Date: Thu, 11 May 2017 13:57:36 -0600 Subject: [PATCH 257/312] TRQ-3991. Make the node health check not inherit pbs_mom's file desriptors. --- src/lib/Libnet/net_server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Libnet/net_server.c b/src/lib/Libnet/net_server.c index 4de9877c32..591cbfa910 100644 --- a/src/lib/Libnet/net_server.c +++ b/src/lib/Libnet/net_server.c @@ -397,7 +397,7 @@ int init_network( if (port != 0) { - sock = socket(AF_INET, SOCK_STREAM, 0); + sock = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (sock < 0) { @@ -449,7 +449,7 @@ int init_network( { /* setup unix domain socket */ - unixsocket = socket(AF_UNIX, SOCK_STREAM, 0); + unixsocket = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); if (unixsocket < 0) { From 4d97ca6ea214b13e2c99c6f8d5aeb601265e453d Mon Sep 17 00:00:00 2001 From: David Beer Date: Mon, 15 May 2017 14:42:31 -0600 Subject: [PATCH 258/312] TRQ-4036. Add a null check when inspecting an attribute. --- src/lib/Libattr/attr_fn_arst.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/Libattr/attr_fn_arst.c b/src/lib/Libattr/attr_fn_arst.c index 811410f6c1..c63281637e 100644 --- a/src/lib/Libattr/attr_fn_arst.c +++ b/src/lib/Libattr/attr_fn_arst.c @@ -1135,7 +1135,9 @@ char *arst_string( return(NULL); } - if ((pattr->at_type != ATR_TYPE_ARST) || !(pattr->at_flags & ATR_VFLAG_SET)) + if ((pattr->at_type != ATR_TYPE_ARST) || + !(pattr->at_flags & ATR_VFLAG_SET) || + (pattr->at_val.at_arst == NULL)) { /* bad type or value not set */ From 34fc9c25d12cbe96603ee557b504fe5a8477b69d Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 16 May 2017 11:13:27 -0600 Subject: [PATCH 259/312] TRQ-4047. Fix another crash around memory only nodes. --- src/lib/Libutils/numa_chip.cpp | 126 ++++++++++++--------------------- src/test/numa_chip/test_uut.c | 51 ++++++++++++- 2 files changed, 97 insertions(+), 80 deletions(-) diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index 2ba3f5c1d9..2e9447493f 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -1271,7 +1271,6 @@ bool Chip::getOpenThreadVector( int execution_slots_per_task) { - unsigned int j = 0; int i = execution_slots_per_task; bool fits = false; @@ -1280,24 +1279,21 @@ bool Chip::getOpenThreadVector( are allocated */ if (execution_slots_per_task == 0) return(true); - slots.clear(); - i = execution_slots_per_task; - j = 0; + /* Can't get contiguous threads. Just get them where you can find them */ // Get the thread indices we will use - do + for (size_t core_index = 0; core_index < this->cores.size() && i != 0; core_index++) { - for (unsigned int x = 0; x < this->cores[j].indices.size(); x++) + for (size_t thread_index = 0; + thread_index < this->cores[core_index].indices.size(); + thread_index++) { - int thread_index; - if (this->cores[j].is_index_busy[x] == true) + if (this->cores[core_index].is_index_busy[thread_index] == true) continue; - thread_index = this->cores[j].indices[x]; - - slots.push_back(thread_index); + slots.push_back(this->cores[core_index].indices[thread_index]); i--; - if ((i == 0) || ((x + 1) == this->cores[j].indices.size())) + if ((i == 0) || ((thread_index + 1) == this->cores[core_index].indices.size())) { /* We fit if all of the execution slots have been filled or it we have used all the chip */ @@ -1305,12 +1301,14 @@ bool Chip::getOpenThreadVector( break; } } - j++; - }while((i != 0) && (j < this->cores.size())); - + } + return(fits); - } + } // END getOpenThreadVector() + + + /* * getContiguousThreadVector * @@ -1328,8 +1326,7 @@ bool Chip::getContiguousThreadVector( int execution_slots_per_task) { - unsigned int j = 0; - int i = execution_slots_per_task; + int i = execution_slots_per_task; bool fits = false; /* this makes it so users can request gpus and mics @@ -1339,15 +1336,15 @@ bool Chip::getContiguousThreadVector( return(true); /* First try to get contiguous threads */ - do + for (size_t core_index = 0; core_index < this->cores.size() && i != 0; core_index++) { - for (unsigned int x = 0; x < this->cores[j].indices.size(); x++) + for (size_t thread_index = 0; + thread_index < this->cores[core_index].indices.size(); + thread_index++) { - int thread_index; - /* if this thread is busy and we have already started creating a list, clear the list and start over; otherwise, continue to the next thread */ - if (this->cores[j].is_index_busy[x] == true) + if (this->cores[core_index].is_index_busy[thread_index] == true) { if (slots.size() > 0) { @@ -1358,9 +1355,7 @@ bool Chip::getContiguousThreadVector( continue; } - thread_index = this->cores[j].indices[x]; - - slots.push_back(thread_index); + slots.push_back(this->cores[core_index].indices[thread_index]); i--; if (i == 0) { @@ -1369,45 +1364,18 @@ bool Chip::getContiguousThreadVector( break; } } - j++; - - }while((i != 0) && (j < this->cores.size())); + } if (fits == false) { slots.clear(); - i = execution_slots_per_task; - j = 0; - /* Can't get contiguous threads. Just get them where you can find them */ - // Get the thread indices we will use - do - { - for (unsigned int x = 0; x < this->cores[j].indices.size(); x++) - { - int thread_index; - if (this->cores[j].is_index_busy[x] == true) - continue; - - thread_index = this->cores[j].indices[x]; - - slots.push_back(thread_index); - i--; - if ((i == 0) || ((x + 1) == this->cores[j].indices.size())) - { - /* We fit if all of the execution slots have been filled - or it we have used all the chip */ - fits = true; - break; - } - } - j++; - - }while((i != 0) && (j < this->cores.size())); + fits = getOpenThreadVector(slots, execution_slots_per_task); } return(fits); } + /* * getContiguousCoreVector * @@ -1425,9 +1393,8 @@ bool Chip::getContiguousCoreVector( int execution_slots_per_task) { - unsigned int j = 0; - int i = execution_slots_per_task; - bool fits = false; + int i = execution_slots_per_task; + bool fits = true; /* this makes it so users can request gpus and mics from numanodes which are not where the cores or threads @@ -1436,50 +1403,51 @@ bool Chip::getContiguousCoreVector( return(true); /* First try to get contiguous cores */ - do + for (size_t core_index = 0; core_index < this->cores.size() && i != 0; core_index++) { - if (this->cores[j].is_free() == true) + if (this->cores[core_index].is_free() == true) { - slots.push_back(j); + slots.push_back(core_index); i--; - j++; - if ((i ==0) || (j == this->cores.size())) - { - /* We fit if all of the execution slots have been filled - or it we have used all the chip */ - fits = true; - } } else { - i = execution_slots_per_task; - j++; + if (i != execution_slots_per_task) + { + fits = false; + i = execution_slots_per_task; + } + slots.clear(); } - }while((i != 0) && (j < this->cores.size())); + } + + if (i == execution_slots_per_task) + fits = false; if (fits == false) { /* Can't get contiguous cores. Just get them where you can find them */ // Get the core indices we will use - j = 0; + size_t core_index = 0; for (int i = 0; i < execution_slots_per_task; i++) { - while (j < this->cores.size()) + while (core_index < this->cores.size()) { - if (this->cores[j].is_free() == true) + if (this->cores[core_index].is_free() == true) { - slots.push_back(j); - j++; + slots.push_back(core_index); + core_index++; break; } else - j++; + core_index++; } } } + return(fits); - } + } // END getContiguousCoreVector() /* diff --git a/src/test/numa_chip/test_uut.c b/src/test/numa_chip/test_uut.c index eb9bbdd729..9166401b6c 100644 --- a/src/test/numa_chip/test_uut.c +++ b/src/test/numa_chip/test_uut.c @@ -17,13 +17,57 @@ extern Json::Value alloc_json; extern void setAllocJson(int); +START_TEST(test_get_contiguous_thread_vector) + { + Chip c; + std::vector list; + + // Make sure a memory only node doesn't crash + fail_unless(c.getContiguousThreadVector(list, 34) == false); + + c.setId(0); + c.setThreads(32); + c.setCores(16); + c.setMemory(6); + c.setChipAvailable(true); + for (int i = 0; i < 16; i++) + c.make_core(i); + + fail_unless(c.getContiguousThreadVector(list, 34) == true); + fail_unless(list.size() == 32); + } +END_TEST + + +START_TEST(test_get_contiguous_core_vector) + { + Chip c; + std::vector list; + + // Make sure a memory only node doesn't crash + fail_unless(c.getContiguousCoreVector(list, 18) == false); + + c.setId(0); + c.setThreads(32); + c.setCores(16); + c.setMemory(6); + c.setChipAvailable(true); + for (int i = 0; i < 16; i++) + c.make_core(i); + + fail_unless(c.getContiguousCoreVector(list, 18) == true); + fail_unless(list.size() == 16); + } +END_TEST + + START_TEST(test_initialize_cores_from_strings) { Chip c; std::string cores; std::string threads; - // Make sure this doesn't segfault + // Make sure this doesn't segfault for a memory only node c.initialize_cores_from_strings(cores, threads); fail_unless(c.getTotalCores() == 0); fail_unless(c.getTotalThreads() == 0); @@ -1261,6 +1305,11 @@ Suite *numa_socket_suite(void) tcase_add_test(tc_core, test_initialize_allocation); tcase_add_test(tc_core, test_place_tasks_execution_slots); suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_get_contiguous_core_vector"); + tcase_add_test(tc_core, test_get_contiguous_core_vector); + tcase_add_test(tc_core, test_get_contiguous_thread_vector); + suite_add_tcase(s, tc_core); return(s); } From 827d2f1e6836a6d7c6e5e99715d0dc1d665fa5cc Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 16 May 2017 15:39:29 -0600 Subject: [PATCH 260/312] TRQ-3995. Take step two for parsing the taskstats file (RUR) for job usage information on the Cray. --- src/include/mom_config.h | 1 + src/include/pbs_job.h | 12 ++ src/resmom/catch_child.c | 13 +- src/resmom/cray_taskstats.cpp | 225 ++++++++++++++++++++++------- src/resmom/mom_main.c | 7 +- src/resmom/parse_config.c | 21 +++ src/test/cray_taskstats/test_uut.c | 62 ++++++-- 7 files changed, 274 insertions(+), 67 deletions(-) diff --git a/src/include/mom_config.h b/src/include/mom_config.h index bdb2b2c7d3..6d44a3d9e5 100644 --- a/src/include/mom_config.h +++ b/src/include/mom_config.h @@ -177,6 +177,7 @@ extern int MOMJobDirStickySet; extern std::string presetup_prologue; extern unsigned long max_memory; extern unsigned long max_swap; +extern bool get_cray_taskstats; struct specials { diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index cacd66ec9e..709292e638 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -530,6 +530,18 @@ typedef std::set job_pid_set_t; #ifdef PBS_MOM // forward declare task so it can be part of the job class task; + +class task_usage_info + { + public: + + unsigned long cput; + unsigned long long mem; + + task_usage_info() : cput(0), mem(0) + { + } + }; #endif #define COUNTED_GLOBALLY 0x0001 diff --git a/src/resmom/catch_child.c b/src/resmom/catch_child.c index 53ff707b7f..195111e36c 100644 --- a/src/resmom/catch_child.c +++ b/src/resmom/catch_child.c @@ -108,6 +108,7 @@ int send_back_std_and_staged_files(job *pjob, int exit_status); /* END external prototypes */ +void update_jobs_usage(job *pjob); void exit_mom_job(job *pjob, int mom_radix); /* @@ -1202,9 +1203,17 @@ int send_job_obit( resc_access_perm = ATR_DFLAG_RDACC; - if (check_rur == true) + if (is_login_node == TRUE) { - get_energy_used(pjob); + if (check_rur == true) + { + get_energy_used(pjob); + } + + if (get_cray_taskstats) + { + update_jobs_usage(pjob); + } } encode_used(pjob, resc_access_perm, NULL, &preq->rq_ind.rq_jobobit.rq_attr); diff --git a/src/resmom/cray_taskstats.cpp b/src/resmom/cray_taskstats.cpp index 0d6b32a1ec..ad505d412b 100644 --- a/src/resmom/cray_taskstats.cpp +++ b/src/resmom/cray_taskstats.cpp @@ -1,6 +1,9 @@ #include #include +#include +#include +#include #include #include @@ -9,6 +12,41 @@ #include "pbs_error.h" #include "resmon.h" +unsigned int total_chars_read = 0; +ssize_t last_line = 0; +std::vector task_lines; +// usage_information maps the job id to the usage reported +std::map usage_information; + + + +/* + * update_usage_information() + * + * Updates our map with the latest and greatest usage information + */ + +void update_usage_information( + + std::string &jobid, + task_usage_info &tui) + + { + std::map::iterator it = usage_information.find(jobid); + + if (it != usage_information.end()) + { + if (tui.cput > it->second.cput) + usage_information[jobid].cput = tui.cput; + + if (tui.mem > it->second.mem) + usage_information[jobid].mem = tui.mem; + } + else + usage_information[jobid] = tui; + } // END update_usage_information() + + /* * parse_rur_line() * @@ -23,81 +61,160 @@ int parse_rur_line( - const std::string &line, - job *pjob) + const std::string &line, + std::set &jobs_this_iteration) { - size_t pos = line.find("taskstats"); + size_t pos = line.find("jobid: "); if (pos != std::string::npos) { - if (line.at(pos) == ' ') - pos++; - if (line.at(pos) == '[') - pos++; - - std::string stats = line.substr(pos, std::string::npos); - char *work_str = strdup(stats.c_str()); - char *ptr = strstr(work_str, "utime',"); - long cput = 0; - long long mem = 0; - resource_def *rd; - resource *pres; + std::string jobid(line.substr(pos + 7)); // + 7 to pass 'jobid: ' + pos = jobid.find(","); - if (ptr != NULL) + if (pos != std::string::npos) { - ptr += 7; // move past "utime'," + jobid.erase(pos); - if (*ptr == ' ') - ptr++; + if (jobs_this_iteration.find(jobid) != jobs_this_iteration.end()) + { + // We're reading the file from the end, or most recent first, so there is no reason to + // look at a second entry for a job. + return(PBSE_NONE); + } - cput = strtol(ptr, &ptr, 10); - } + pos = line.find("taskstats ["); - if ((ptr = strstr(ptr, "stime',")) != NULL) - { - ptr += 7; // move past "stime'," - - if (*ptr == ' ') - ptr++; + if (pos != std::string::npos) + { + // Add 11 for 'taskstats [' + std::string stats = line.substr(pos + 11, std::string::npos); + char *work_str = strdup(stats.c_str()); + char *ptr = work_str; + char *tmp_ptr = strstr(ptr, "utime',"); + task_usage_info tui; - cput += strtol(ptr, &ptr, 10); - } + if (tmp_ptr != NULL) + { + ptr = tmp_ptr + 7; // move past "utime'," - if (cput > 0) - { - rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); - pres = add_resource_entry(pjob->ji_wattr + JOB_ATR_resc_used, rd); - pres->rs_value.at_flags |= ATR_VFLAG_SET; - pres->rs_value.at_type = ATR_TYPE_LONG; - pres->rs_value.at_val.at_long = MAX(cput, pres->rs_value.at_val.at_long); - } + if (*ptr == ' ') + ptr++; - if ((ptr = strstr(ptr, "max_rss',")) != NULL) - { - ptr += 9; // move past "max_rss'," + tui.cput = strtol(ptr, &ptr, 10); + } - if (*ptr == ' ') - ptr++; + if ((tmp_ptr = strstr(ptr, "stime',")) != NULL) + { + ptr = tmp_ptr + 7; // move past "stime'," + + if (*ptr == ' ') + ptr++; - mem = strtoll(ptr, NULL, 10); + tui.cput += strtol(ptr, &ptr, 10); + } - if (mem > 0) - { - rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); - pres = add_resource_entry(pjob->ji_wattr + JOB_ATR_resc_used, rd); - pres->rs_value.at_flags |= ATR_VFLAG_SET; - pres->rs_value.at_type = ATR_TYPE_SIZE; - pres->rs_value.at_val.at_size.atsv_shift = 10; - pres->rs_value.at_val.at_size.atsv_units = ATR_SV_BYTESZ; - pres->rs_value.at_val.at_size.atsv_num = MAX(mem, pres->rs_value.at_val.at_size.atsv_num); + if ((ptr = strstr(ptr, "max_rss',")) != NULL) + { + ptr += 9; // move past "max_rss'," + + if (*ptr == ' ') + ptr++; + + tui.mem = strtoll(ptr, NULL, 10); + } + + jobs_this_iteration.insert(jobid); + update_usage_information(jobid, tui); + + free(work_str); } } - - free(work_str); } + return(PBSE_NONE); } // parse_rur_line() + +/* + * append_date() + * + * Appends the date format to the end of the task_stats path + */ + +void append_date( + + std::string &path) + + { + // NYI: find out the date format and append it + // NYI: add logic to notice when it is a new day + } // END append_date() + + + +void read_rur_stats_file( + + const char *basepath) + + { + std::string path(basepath); + + append_date(path); + + std::ifstream task_file(path.c_str()); + + if (task_file.is_open()) + { + std::set jobs_this_iteration; + task_file.seekg(total_chars_read); + last_line = task_lines.size(); + + // Read in all new lines + for (std::string line; std::getline(task_file, line); ) + { + total_chars_read += line.size() + 1; // Add one for the newline that gets eaten. + task_lines.push_back(line); + } + + for (ssize_t i = task_lines.size() - 1; i >= last_line; i--) + parse_rur_line(task_lines[i], jobs_this_iteration); + } + } // END read_rur_stats_file() + + + +void update_jobs_usage( + + job *pjob) + + { + std::string numeric_jobid(pjob->ji_qs.ji_jobid); + size_t pos = numeric_jobid.find("."); + if (pos != std::string::npos) + numeric_jobid.erase(pos); + + std::map::iterator it = usage_information.find(numeric_jobid); + if (it != usage_information.end()) + { + resource_def *rd; + resource *pres; + + rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); + pres = add_resource_entry(pjob->ji_wattr + JOB_ATR_resc_used, rd); + pres->rs_value.at_flags |= ATR_VFLAG_SET; + pres->rs_value.at_type = ATR_TYPE_LONG; + pres->rs_value.at_val.at_long = MAX(it->second.cput, pres->rs_value.at_val.at_long); + + rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); + pres = add_resource_entry(pjob->ji_wattr + JOB_ATR_resc_used, rd); + pres->rs_value.at_flags |= ATR_VFLAG_SET; + pres->rs_value.at_type = ATR_TYPE_SIZE; + pres->rs_value.at_val.at_size.atsv_shift = 10; + pres->rs_value.at_val.at_size.atsv_units = ATR_SV_BYTESZ; + pres->rs_value.at_val.at_size.atsv_num = MAX(it->second.mem, pres->rs_value.at_val.at_size.atsv_num); + } + } // END update_jobs_usage() + diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index c5afa90a7e..56dee522d4 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -279,9 +279,10 @@ int use_nvidia_gpu = TRUE; pjobexec_t TMOMStartInfo[TMAX_JE]; - +const char *taskstats_basepath = "/var/opt/cray/log/partition-current/messages-"; /* prototypes */ +void read_rur_stats_file(const char *basepath); void read_mom_hierarchy(); void sort_paths(); void resend_things(); @@ -6656,6 +6657,10 @@ void main_loop(void) { MOMCheckRestart(); /* There are no jobs, see if the server needs to be restarted. */ } + + if (get_cray_taskstats) + read_rur_stats_file(taskstats_basepath); + } /* END while (mom_run_state == MOM_RUN_STATE_RUNNING) */ #ifdef ENABLE_PMIX diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index bc972e8450..4743741a34 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -199,6 +199,7 @@ int mom_hierarchy_retry_time = NODE_COMM_RETRY_TIME; std::string presetup_prologue; unsigned long max_memory = 0; unsigned long max_swap = 0; +bool get_cray_taskstats = false; @@ -311,6 +312,7 @@ unsigned long setcudavisibledevices(const char *); unsigned long set_presetup_prologue(const char *); unsigned long set_max_physical_memory(const char *); unsigned long set_max_swap_memory(const char *); +unsigned long set_get_cray_taskstats(const char *); struct specials special[] = { { "force_overwrite", setforceoverwrite}, @@ -398,6 +400,7 @@ struct specials special[] = { { "presetup_prologue", set_presetup_prologue}, { "max_physical_memory", set_max_physical_memory}, { "max_swap_memory", set_max_swap_memory}, + { "get_cray_taskstats", set_get_cray_taskstats}, { NULL, NULL } }; @@ -573,6 +576,24 @@ unsigned long set_max_swap_memory( } + +unsigned long set_get_cray_taskstats( + + const char *value) + + { + int enable; + + if ((enable = setbool(value)) != -1) + { + get_cray_taskstats = (bool)enable; + return(1); + } + + return(0); /* error */ + } + + unsigned long setidealload( const char *value) diff --git a/src/test/cray_taskstats/test_uut.c b/src/test/cray_taskstats/test_uut.c index c08563472b..24833a58cc 100644 --- a/src/test/cray_taskstats/test_uut.c +++ b/src/test/cray_taskstats/test_uut.c @@ -9,16 +9,27 @@ #include -int parse_rur_line(const std::string &line, job *pjob); +int parse_rur_line(const std::string &line, std::set &jobs_this_iteration); +void update_jobs_usage(job *pjob); +void read_rur_stats_file(const char *basepath); + const char *l1 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 100, 'stime', 100, 'max_rss', 1984, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; const char *l2 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 300, 'stime', 0, 'max_rss', 4444, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; +const char *l3 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 1, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 300, 'stime', 300, 'max_rss', 555555, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; +const char *l4 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 2, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 400, 'stime', 300, 'max_rss', 5, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; + + START_TEST(test_parse_rur_line) { - job pjob; + job pjob; + std::set jobs_this_iteration; memset(&pjob, 0, sizeof(pjob)); - fail_unless(parse_rur_line(l1, &pjob) == PBSE_NONE); + snprintf(pjob.ji_qs.ji_jobid, sizeof(pjob.ji_qs.ji_jobid), "0.roshar"); + + fail_unless(parse_rur_line(l1, jobs_this_iteration) == PBSE_NONE); + update_jobs_usage(&pjob); resource_def *rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); resource *pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); fail_unless(pres != NULL); @@ -31,7 +42,9 @@ START_TEST(test_parse_rur_line) "%llu", pres->rs_value.at_val.at_size.atsv_num); // Make sure we increase the value if we are using more later - fail_unless(parse_rur_line(l2, &pjob) == PBSE_NONE); + jobs_this_iteration.clear(); + fail_unless(parse_rur_line(l2, jobs_this_iteration) == PBSE_NONE); + update_jobs_usage(&pjob); rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); fail_unless(pres != NULL); @@ -43,7 +56,9 @@ START_TEST(test_parse_rur_line) fail_unless(pres->rs_value.at_val.at_size.atsv_num == 4444); // Make sure the values don't decrease if we find less later - fail_unless(parse_rur_line(l1, &pjob) == PBSE_NONE); + jobs_this_iteration.clear(); + fail_unless(parse_rur_line(l1, jobs_this_iteration) == PBSE_NONE); + update_jobs_usage(&pjob); rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); fail_unless(pres != NULL); @@ -56,13 +71,40 @@ START_TEST(test_parse_rur_line) } END_TEST -START_TEST(test_two) - { - +START_TEST(test_read_rur_stats_file) + { + const char *file = "sample_file"; + extern std::map usage_information; + std::map::iterator it; + + usage_information.clear(); + + read_rur_stats_file(file); + + it = usage_information.find("0"); + fail_unless(it != usage_information.end()); + fail_unless(it->second.cput == 30); + fail_unless(it->second.mem == 30); + + it = usage_information.find("1"); + fail_unless(it != usage_information.end()); + fail_unless(it->second.cput == 1); + fail_unless(it->second.mem == 2); + + it = usage_information.find("2"); + fail_unless(it != usage_information.end()); + fail_unless(it->second.cput == 13); + fail_unless(it->second.mem == 8); + + it = usage_information.find("3"); + fail_unless(it != usage_information.end()); + fail_unless(it->second.cput == 8); + fail_unless(it->second.mem == 9); } END_TEST + Suite *cray_taskstats_suite(void) { Suite *s = suite_create("cray_taskstats_suite methods"); @@ -70,8 +112,8 @@ Suite *cray_taskstats_suite(void) tcase_add_test(tc_core, test_parse_rur_line); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_read_rur_stats_file"); + tcase_add_test(tc_core, test_read_rur_stats_file); suite_add_tcase(s, tc_core); return s; From c536e7f7c5d96d9016d4d5bafde4b0f4488144dd Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 16 May 2017 16:21:05 -0600 Subject: [PATCH 261/312] TRQ-3995. Add the sample file for the unit test. --- src/test/cray_taskstats/sample_file | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/test/cray_taskstats/sample_file diff --git a/src/test/cray_taskstats/sample_file b/src/test/cray_taskstats/sample_file new file mode 100644 index 0000000000..6ea36d3152 --- /dev/null +++ b/src/test/cray_taskstats/sample_file @@ -0,0 +1,5 @@ +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 3, 'stime', 4, 'max_rss', 5, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 1, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 0, 'stime', 1, 'max_rss', 2, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 2, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 6, 'stime', 7, 'max_rss', 8, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 10, 'stime', 20, 'max_rss', 30, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 3, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 4, 'stime', 4, 'max_rss', 9, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] From 4eed6f1bc07c54b2548364eb943e406f90b241fb Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 16 May 2017 16:59:08 -0600 Subject: [PATCH 262/312] TRQ-1990. Make -fstack-protector the default. --- configure.ac | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 124aee8da0..848893cf5d 100644 --- a/configure.ac +++ b/configure.ac @@ -1487,11 +1487,12 @@ AC_ARG_ENABLE(align_memory, [ --enable-align-memory Change compile flags to get rid of unaligned memory access errors]) if test "x$GCC" = "xyes" ;then AC_MSG_CHECKING([whether add memory alignment flags]) - if test "${enable_align_memory}" = "yes" ; then - AC_MSG_RESULT([yes]) - CFLAGS="$CFLAGS -fstack-protector -Wformat -Wformat-security -DFORTIFY_SOURCE=2" - else + if test "$x{enable_align_memory}" = "xno" ; then AC_MSG_RESULT([no]) + else + AC_MSG_RESULT([default yes]) + CFLAGS="$CFLAGS -fstack-protector -Wformat -Wformat-security -DFORTIFY_SOURCE=2" + CXXFLAGS="$CXXFLAGS -fstack-protector -Wformat -Wformat-security -DFORTIFY_SOURCE=2" fi fi From b435d72b08f11872191d5a89987da786147d3461 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 16 May 2017 17:03:58 -0600 Subject: [PATCH 263/312] TRQ-3995. Add some scaffolding definitions. --- src/test/catch_child/scaffolding.c | 4 ++++ src/test/mom_main/scaffolding.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/test/catch_child/scaffolding.c b/src/test/catch_child/scaffolding.c index 11ba520251..92af599383 100644 --- a/src/test/catch_child/scaffolding.c +++ b/src/test/catch_child/scaffolding.c @@ -1422,3 +1422,7 @@ task::~task() {} batch_request::batch_request(int type) : rq_type(type) { } + +bool get_cray_taskstats; + +void update_jobs_usage(job *pjob) {} diff --git a/src/test/mom_main/scaffolding.c b/src/test/mom_main/scaffolding.c index b02611263b..2da9eece4a 100644 --- a/src/test/mom_main/scaffolding.c +++ b/src/test/mom_main/scaffolding.c @@ -1143,3 +1143,6 @@ int read_all_devices() return(PBSE_NONE); } +bool get_cray_taskstats; + +void read_rur_stats_file(const char *basepath) {} From 5c2b7ed4bae2ab9811ed045313fe3e7ad89b6a55 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 17 May 2017 16:46:39 -0600 Subject: [PATCH 264/312] TRQ-3978. When hwloc fails to tell us where GPUs are, add them to socket 0, node 0. We discovered a bug in hwloc that sometimes prevents it from reporting GPUs that are visible in lspci and nvidia-smi. This is the workaround for it. We will try to get a bug opened with hwloc. --- src/include/machine.hpp | 2 +- src/lib/Libutils/machine.cpp | 6 ++++-- src/lib/Libutils/numa_pci_device.cpp | 19 ++++++++++++------ src/resmom/accelerators_numa.cpp | 27 ++++++++++++++++++++------ src/test/machine/test_uut.c | 4 ++-- src/test/numa_pci_device/scaffolding.c | 2 +- 6 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/include/machine.hpp b/src/include/machine.hpp index dfae0010f2..845f7d5a71 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -358,7 +358,7 @@ class Machine void displayAsString(stringstream &out) const; void displayAsJson(stringstream &out, bool include_jobs) const; void insertNvidiaDevice(PCI_Device& device); - void store_device_on_appropriate_chip(PCI_Device &device); + void store_device_on_appropriate_chip(PCI_Device &device, bool no_info); void place_all_execution_slots(req &r, allocation &master, const char *hostname); int spread_place(req &r, allocation &master, int tasks_for_node, const char *hostname); int spread_place_pu(req &r, allocation &master, int tasks_for_node, const char *hostname); diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index 0b7f86d0eb..0e191f66a6 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -1304,10 +1304,12 @@ void Machine::free_job_allocation( void Machine::store_device_on_appropriate_chip( - PCI_Device &device) + PCI_Device &device, + bool no_info) { - if (this->isNUMA == false) + if ((this->isNUMA == false) || + (no_info == true)) { this->sockets[0].store_pci_device_appropriately(device, true); } diff --git a/src/lib/Libutils/numa_pci_device.cpp b/src/lib/Libutils/numa_pci_device.cpp index 76e0e2eccd..bf8dcdd960 100644 --- a/src/lib/Libutils/numa_pci_device.cpp +++ b/src/lib/Libutils/numa_pci_device.cpp @@ -68,13 +68,20 @@ PCI_Device::~PCI_Device() int PCI_Device::initializePCIDevice(hwloc_obj_t device_obj, int idx, hwloc_topology_t topology) { - - id = device_obj->logical_index; - name = device_obj->name; - if (device_obj->infos != NULL) + if (device_obj != NULL) + { + id = device_obj->logical_index; + name = device_obj->name; + if (device_obj->infos != NULL) + { + info_name = device_obj->infos->name; + info_value = device_obj->infos->value; + } + } + else { - info_name = device_obj->infos->name; - info_value = device_obj->infos->value; + id = idx; + name = "Unknown"; } diff --git a/src/resmom/accelerators_numa.cpp b/src/resmom/accelerators_numa.cpp index ddf9230ed3..0d2ff498d7 100644 --- a/src/resmom/accelerators_numa.cpp +++ b/src/resmom/accelerators_numa.cpp @@ -86,7 +86,11 @@ hwloc_obj_t Machine::get_non_nvml_device( -int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t topology) +int Machine::initializeNVIDIADevices( + + hwloc_obj_t machine_obj, + hwloc_topology_t topology) + { nvmlReturn_t rc; @@ -103,8 +107,9 @@ int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t t return(PBSE_NONE); } - unsigned int device_count = 0; - unsigned int found_devices = 0; + std::vector unfound_ids; + unsigned int device_count = 0; + unsigned int found_devices = 0; /* Get the device count. */ rc = nvmlDeviceGetCount(&device_count); @@ -136,7 +141,10 @@ int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t t // including k80s. gpu_obj = this->get_non_nvml_device(topology, gpu, identified); if (gpu_obj == NULL) + { + unfound_ids.push_back(idx); continue; + } } identified.insert(gpu_obj); @@ -153,14 +161,14 @@ int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t t new_device.initializePCIDevice(gpu_obj, idx, topology); - store_device_on_appropriate_chip(new_device); + store_device_on_appropriate_chip(new_device, false); // hwloc sees the K80 as a single device. We want to display two to stay in sync // with NVML, so store an extra for each K80 if (!strncmp(buf, "Tesla K80", 9)) { new_device.setId(new_device.get_id() + 1); - store_device_on_appropriate_chip(new_device); + store_device_on_appropriate_chip(new_device, false); found_devices++; idx++; } @@ -169,9 +177,16 @@ int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t t if (found_devices != device_count) { - sprintf(log_buffer, "NVML reports %u devices, but we only found %u", + sprintf(log_buffer, "NVML reports %u devices, but we only found %u. Placement will be approximated.", device_count, found_devices); log_err(-1, __func__, log_buffer); + + for (size_t i = 0; i < unfound_devices.size(); i++) + { + PCI_Device new_device; + new_device.initializePCIDevice(NULL, unfound_devices[i], topology); + store_device_on_appropriate_chip(new_device, true); + } } } else diff --git a/src/test/machine/test_uut.c b/src/test/machine/test_uut.c index 02de9daf32..3846bcc3a9 100644 --- a/src/test/machine/test_uut.c +++ b/src/test/machine/test_uut.c @@ -219,13 +219,13 @@ START_TEST(test_store_pci_device_on_appropriate_chip) // Since this is non numa it should be forced to socket 0 m.setIsNuma(false); - m.store_device_on_appropriate_chip(d); + m.store_device_on_appropriate_chip(d, true); fail_unless(called_store_pci == 1); // Since this is numa it should place on the sockets until it returns true, // which due to the scaffolding is never, so once per socket m.setIsNuma(true); - m.store_device_on_appropriate_chip(d); + m.store_device_on_appropriate_chip(d, false); fail_unless(called_store_pci == 3); } END_TEST diff --git a/src/test/numa_pci_device/scaffolding.c b/src/test/numa_pci_device/scaffolding.c index f3bbcf198f..719276333c 100644 --- a/src/test/numa_pci_device/scaffolding.c +++ b/src/test/numa_pci_device/scaffolding.c @@ -38,7 +38,7 @@ void log_nvml_error(nvmlReturn_t rc, char* gpuid, const char* id) } #endif -void Machine::store_device_on_appropriate_chip( PCI_Device &device) +void Machine::store_device_on_appropriate_chip(PCI_Device &device, bool no_info) { return; } From 3b999a82c94f41a6246e137222db2a49d3aebbdd Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 17 May 2017 16:48:14 -0600 Subject: [PATCH 265/312] TRQ-4046. Don't unlock and unlocked node in write_node_state_work(). This causes crashes in newer versions of the pthread library. --- src/server/node_manager.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index 446bb765cc..dc7dfcbf8a 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -1573,8 +1573,6 @@ void *write_node_state_work( { fprintf(nstatef, fmt, np->get_name(), np->nd_state & savemask); } - - np->unlock_node(__func__, NULL, LOGLEVEL); } /* END for each node */ } else From ed282d3e93764efc6a1fd236664f365337d64d21 Mon Sep 17 00:00:00 2001 From: David Beer Date: Wed, 17 May 2017 16:57:39 -0600 Subject: [PATCH 266/312] TRQ-3978. Fix an error in the previous patch. --- src/resmom/accelerators_numa.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resmom/accelerators_numa.cpp b/src/resmom/accelerators_numa.cpp index 0d2ff498d7..e890f0aafe 100644 --- a/src/resmom/accelerators_numa.cpp +++ b/src/resmom/accelerators_numa.cpp @@ -181,10 +181,10 @@ int Machine::initializeNVIDIADevices( device_count, found_devices); log_err(-1, __func__, log_buffer); - for (size_t i = 0; i < unfound_devices.size(); i++) + for (size_t i = 0; i < unfound_ids.size(); i++) { PCI_Device new_device; - new_device.initializePCIDevice(NULL, unfound_devices[i], topology); + new_device.initializePCIDevice(NULL, unfound_ids[i], topology); store_device_on_appropriate_chip(new_device, true); } } From a5b54aa063f55b189ee2c5682768730651c05ad5 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 22 May 2017 15:24:22 -0600 Subject: [PATCH 267/312] TRQ-4017 - fix NVIDIA GPU assignment indexing problem --- src/include/trq_cgroups.h | 1 + src/resmom/mom_server.c | 4 +- src/resmom/mom_server_lib.h | 2 +- src/resmom/nvidia.c | 207 ++++++++++++++++++++++------- src/test/mom_server/test_uut.c | 4 +- src/test/nvidia/scaffolding.c | 134 ++++++++++++++++++- src/test/nvidia/test_uut.c | 27 ++++ src/test/trq_cgroups/scaffolding.c | 2 +- 8 files changed, 329 insertions(+), 52 deletions(-) diff --git a/src/include/trq_cgroups.h b/src/include/trq_cgroups.h index f765ef58e3..d140c890dc 100644 --- a/src/include/trq_cgroups.h +++ b/src/include/trq_cgroups.h @@ -41,4 +41,5 @@ bool have_incompatible_dash_l_resource(pbs_attribute *pattr); int trq_cg_add_devices_to_cgroup(job *pjob); int init_torque_cgroups(); void trq_cg_signal_tasks(const std::string& cgroup_path, int signal); +int find_range_in_cpuset_string(std::string &source, std::string &output); #endif /* _TRQ_CGROUPS_H_ */ diff --git a/src/resmom/mom_server.c b/src/resmom/mom_server.c index 493bd7ce6b..f93e01f2ab 100644 --- a/src/resmom/mom_server.c +++ b/src/resmom/mom_server.c @@ -3444,7 +3444,7 @@ bool is_for_this_host( void get_device_indices( const char *device_str, - std::vector &device_indices, + std::vector &device_indices, const char *suffix) { @@ -3500,7 +3500,7 @@ void get_device_indices( if (is_for_this_host(host_name_part, suffix) == true) { - unsigned int device_index = atoi(device_index_part.c_str()); + int device_index = atoi(device_index_part.c_str()); device_indices.push_back(device_index); } diff --git a/src/resmom/mom_server_lib.h b/src/resmom/mom_server_lib.h index b62204f0e1..cc89cba9a1 100644 --- a/src/resmom/mom_server_lib.h +++ b/src/resmom/mom_server_lib.h @@ -67,7 +67,7 @@ void mom_server_update_gpustat(mom_server *pms, char *status_strings); #endif /* NVIDIA_GPUS */ -void get_device_indices(const char *device_str, std::vector &device_indices, const char *suffix); +void get_device_indices(const char *device_str, std::vector &device_indices, const char *suffix); void generate_server_status(std::vector& status); #ifdef NVML_API diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index f7c8a9f897..23637bc586 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -95,6 +95,7 @@ #include #include #include +#include #include #include #if defined(NTOHL_NEEDS_ARPA_INET_H) && defined(HAVE_ARPA_INET_H) @@ -143,6 +144,9 @@ extern int use_nvidia_gpu; extern time_t time_now; extern unsigned int global_gpu_count; +#ifdef NVML_API +std::map gpu_minor_to_gpu_index; +#endif /* NVML_API */ int nvidia_gpu_modes[50]; #ifdef NUMA_SUPPORT @@ -258,6 +262,86 @@ void log_nvml_error( } } +/** + * build_gpu_minor_to_gpu_index_map() + * + * Create a mapping from the NVIDIA gpu device minor numbers to the index + * numbers used with the NVML library. + * + * Note that the device index ordering used by the NVML library is based on + * the PCI bus ID. Torque server assumes an ordering based on the device + * minor number (see place_subnodes_in_hostlist()). These two orderings are + * not guaranteed to be the same so a mapping is needed to map the minor + * number to the index number used by the library. + * + * @param device_count - the number of gpu devices on this system + * @return -1 failure, 0 success + * + */ + +int build_gpu_minor_to_gpu_index_map( + + unsigned int device_count) + + { + nvmlDevice_t dev_handle; + unsigned int minor; + unsigned int index; + + // clear the map + gpu_minor_to_gpu_index.clear(); + + // build the map + for (index = 0; index < device_count; index++) + { + // get the device handle + if (nvmlDeviceGetHandleByIndex(index, &dev_handle) != NVML_SUCCESS) + { + gpu_minor_to_gpu_index.clear(); + return(-1); + } + + // look up the device handle's minor number + if (nvmlDeviceGetMinorNumber(dev_handle, &minor) != NVML_SUCCESS) + { + gpu_minor_to_gpu_index.clear(); + return(-1); + } + + // map the minor number to the NVML index + gpu_minor_to_gpu_index[minor] = index; + } + + return(0); + } + +/** + * get_gpu_handle_by_minor() + * + * Get NVIDIA device handle by device minor number. + * + * @param minor - the device minor number + * @param device - reference in which to return the device handle + * @return NVML_ERROR_UNKNOWN if minor number not in map, or an nvmlReturn_t value otherwise + * + */ + +nvmlReturn_t get_gpu_handle_by_minor( + + unsigned int minor, + nvmlDevice_t *device) + + { + std::map::const_iterator it + = gpu_minor_to_gpu_index.find(minor); + + // confirm that minor number is in the map + if (it == gpu_minor_to_gpu_index.end()) + return(NVML_ERROR_UNKNOWN); + + // return the device handle + return(nvmlDeviceGetHandleByIndex(it->second, device)); + } /* * Function to initialize the Nvidia nvml api @@ -278,7 +362,16 @@ bool init_nvidia_nvml( if (rc == NVML_SUCCESS) { if ((int)device_count > 0) + { + // build map function for minor_to_gpu_index + if (build_gpu_minor_to_gpu_index_map(device_count) != 0) + { + shut_nvidia_nvml(); + return(false); + } + return(true); + } sprintf(log_buffer,"No Nvidia gpus detected\n"); log_ext(-1, __func__, log_buffer, LOG_DEBUG); @@ -309,6 +402,9 @@ int shut_nvidia_nvml() if (!use_nvidia_gpu) return (TRUE); + // clear the map + gpu_minor_to_gpu_index.clear(); + rc = nvmlShutdown(); if (rc == NVML_SUCCESS) @@ -755,19 +851,17 @@ int setgpumode( } /* get the device handle */ - - rc = nvmlDeviceGetHandleByIndex(gpuid, &device_hndl); + rc = get_gpu_handle_by_minor(gpuid, &device_hndl); if (device_hndl != NULL) { - if (LOGLEVEL >= 7) - { + if (LOGLEVEL >= 7) + { sprintf(log_buffer, "changing to mode %d for gpu %d", - gpumode, - gpuid); + gpumode, gpuid); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - } + } rc = nvmlDeviceSetComputeMode(device_hndl, compute_mode); @@ -920,20 +1014,17 @@ int resetgpuecc( } /* get the device handle */ - - rc = nvmlDeviceGetHandleByIndex(gpuid, &device_hndl); + rc = get_gpu_handle_by_minor(gpuid, &device_hndl); if (device_hndl != NULL) { - if (LOGLEVEL >= 7) - { - sprintf(log_buffer, "reseting error count %d-%d for gpu %d", - reset_perm, - reset_vol, - gpuid); + if (LOGLEVEL >= 7) + { + sprintf(log_buffer, "resetting error count %d-%d for gpu %d", + reset_perm, reset_vol, gpuid); - log_ext(-1, __func__, log_buffer, LOG_DEBUG); - } + log_ext(-1, __func__, log_buffer, LOG_DEBUG); + } rc = nvmlDeviceClearEccErrorCounts(device_hndl, counter_type); @@ -1075,7 +1166,7 @@ int resetgpuecc( int set_gpu_modes( - std::vector &gpu_indices, + std::vector &gpu_indices, int gpu_flags) { @@ -1109,7 +1200,7 @@ int set_gpu_modes( initialized = true; #endif - for (std::vector::iterator it = gpu_indices.begin(); it != gpu_indices.end(); it++) + for (std::vector::iterator it = gpu_indices.begin(); it != gpu_indices.end(); it++) { int set_mode_result; /* check to see if we need to reset error counts */ @@ -1117,7 +1208,7 @@ int set_gpu_modes( { if (LOGLEVEL >= 7) { - sprintf(log_buffer, "reseting gpuid %d volatile error counts", + sprintf(log_buffer, "resetting gpuid %d volatile error counts", *it); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); @@ -1194,14 +1285,14 @@ int get_gpu_mode( int set_gpu_req_modes( - std::vector &gpu_indices, - int gpu_flags, - job *pjob) + std::vector &gpu_indices, + int gpu_flags, + job *pjob) { pbs_attribute *pattr; int rc; - unsigned int gpu_indices_size = gpu_indices.size(); + int gpu_indices_size = gpu_indices.size(); size_t gpu_index = 0; bool initialized = false; @@ -1241,7 +1332,7 @@ int set_gpu_req_modes( initialized = true; #endif - for (unsigned int i = 0; i < cr->get_num_reqs() && gpu_index < gpu_indices.size(); i++) + for (int i = 0; i < cr->get_num_reqs() && gpu_index < gpu_indices.size(); i++) { int total_req_gpus; @@ -1250,7 +1341,7 @@ int set_gpu_req_modes( gpu_mode = r.get_gpu_mode(); /* returns a string indicating the gpu mode */ total_req_gpus = r.get_gpus(); - for (unsigned int j = 0; j < total_req_gpus && gpu_index < gpu_indices.size(); j++) + for (int j = 0; j < total_req_gpus && gpu_index < gpu_indices.size(); j++) { /* only use as many gpus as requested for the req */ int mode; @@ -1323,15 +1414,55 @@ int setup_gpus_for_job( job *pjob) /* I */ { +#ifdef PENABLE_LINUX_CGROUPS + std::string gpus_reserved; + std::string gpu_range; +#else char *gpu_str; +#endif int gpu_flags = 0; int rc; + std::vector gpu_indices; /* if node does not have Nvidia recognized driver version then forget it */ if (MOMNvidiaDriverVersion < 260) return(PBSE_NONE); +#ifdef PENABLE_LINUX_CGROUPS + /* if there are no gpus, do nothing */ + if ((pjob->ji_wattr[JOB_ATR_gpus_reserved].at_flags & ATR_VFLAG_SET) == 0) + return(PBSE_NONE); + + /* if there are no gpu flags, do nothing */ + if ((pjob->ji_wattr[JOB_ATR_gpu_flags].at_flags & ATR_VFLAG_SET) == 0) + return(PBSE_NONE); + + gpus_reserved = pjob->ji_wattr[JOB_ATR_gpus_reserved].at_val.at_str; + + if (gpus_reserved.length() == 0) + return(PBSE_NONE); + + find_range_in_cpuset_string(gpus_reserved, gpu_range); + translate_range_string_to_vector(gpu_range.c_str(), gpu_indices); + + gpu_flags = pjob->ji_wattr[JOB_ATR_gpu_flags].at_val.at_long; + + if (LOGLEVEL >= 7) + { + sprintf(log_buffer, "job %s has gpus_reserved %s gpu_flags %d", + pjob->ji_qs.ji_jobid, gpu_range.c_str(), gpu_flags); + + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + } + + rc = set_gpu_req_modes(gpu_indices, gpu_flags, pjob); + if (rc != PBSE_NONE) + { + sprintf(log_buffer, "Failed to set gpu modes for job %s. error %d", pjob->ji_qs.ji_jobid, rc); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + } +#else /* if there are no gpus, do nothing */ if ((pjob->ji_wattr[JOB_ATR_exec_gpus].at_flags & ATR_VFLAG_SET) == 0) return(PBSE_NONE); @@ -1349,34 +1480,20 @@ int setup_gpus_for_job( if (LOGLEVEL >= 7) { - sprintf(log_buffer, "job %s has exec_gpus %s gpu_flags %d", - pjob->ji_qs.ji_jobid, - gpu_str, - gpu_flags); + sprintf(log_buffer, "job %s has exec_gpus %s gpu_flags %d", + pjob->ji_qs.ji_jobid, gpu_str, gpu_flags); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); } /* tokenize all of the gpu allocations the format of the gpu string is -gpu/index[+-gpu/index...]*/ /* traverse the gpu_str to see what gpus we have assigned */ - - std::vector gpu_indices; get_device_indices(gpu_str, gpu_indices, "-gpu"); -#ifndef PENABLE_LINUX_CGROUPS rc = set_gpu_modes(gpu_indices, gpu_flags); -#else - rc = set_gpu_req_modes(gpu_indices, gpu_flags, pjob); - if (rc != PBSE_NONE) - { - sprintf(log_buffer, "Failed to set gpu modes for job %s. error %d", pjob->ji_qs.ji_jobid, rc); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - } - -#endif /* PENABLE_LINUX_CGROUPS */ - +#endif return(rc); } /* END setup_gpus_for_job() */ @@ -1470,7 +1587,7 @@ void generate_server_gpustatus_nvml( for (idx = 0; idx < (int)device_count; idx++) #endif { - rc = nvmlDeviceGetHandleByIndex(idx, &device_hndl); + rc = get_gpu_handle_by_minor(idx, &device_hndl); if (rc != NVML_SUCCESS) { diff --git a/src/test/mom_server/test_uut.c b/src/test/mom_server/test_uut.c index 0df4eb051f..bdd08001ab 100644 --- a/src/test/mom_server/test_uut.c +++ b/src/test/mom_server/test_uut.c @@ -33,7 +33,7 @@ extern int is_reporter_mom; extern mom_server mom_servers[PBS_MAXSERVER]; bool is_for_this_host(std::string gpu_spec, const char *suffix); -void get_device_indices(const char *gpu_str, std::vector &gpu_indices, const char *suffix); +void get_device_indices(const char *gpu_str, std::vector &gpu_indices, const char *suffix); START_TEST(test_sort_paths) { @@ -250,7 +250,7 @@ END_TEST START_TEST(test_get_device_indices) { std::string spec; - std::vector gpu_indices; + std::vector gpu_indices; char suffix[10]; extern char mom_alias[]; diff --git a/src/test/nvidia/scaffolding.c b/src/test/nvidia/scaffolding.c index 28cfe829e8..e20adbf5ef 100644 --- a/src/test/nvidia/scaffolding.c +++ b/src/test/nvidia/scaffolding.c @@ -7,9 +7,18 @@ #include "log.h" #include "mcom.h" #include +#include +#include + +#include "attribute.h" +#include "resource.h" +#include "complete_req.hpp" char log_buffer[LOG_BUF_SIZE]; nvmlReturn_t global_nvmlDeviceGetFanSpeed_rc = NVML_SUCCESS; +nvmlDevice_t global_device; +unsigned int global_device_minor_number = 0; +std::string global_string = "foo"; int MXMLFromString( @@ -31,13 +40,27 @@ void log_event(int eventtype, int objclass, const char *objname, const char *tex void send_update_soon() { return; } -void get_device_indices(const char *device_str, std::vector &device_indices, const char *suffix) {} +void get_device_indices(const char *device_str, std::vector &device_indices, const char *suffix) {} nvmlReturn_t nvmlDeviceGetFanSpeed(nvmlDevice_t device_hndl, unsigned int *i) { return(global_nvmlDeviceGetFanSpeed_rc); } +nvmlReturn_t nvmlDeviceGetHandleByIndex(unsigned int index, nvmlDevice_t *device_hndl) + { + *device_hndl = global_device; + + return(NVML_SUCCESS); + } + +nvmlReturn_t nvmlDeviceGetMinorNumber(nvmlDevice_t device_hndl, unsigned int *minor) + { + *minor = global_device_minor_number++; + + return(NVML_SUCCESS); + } + void log_err( int errnum, /* I (errno or PBSErrno) */ const char *routine, /* I */ @@ -46,3 +69,112 @@ void log_err( { snprintf(log_buffer, sizeof(log_buffer), "%s %s", routine, text); } + + +int is_whitespace( + + char c) + + { + if ((c == ' ') || + (c == '\n') || + (c == '\t') || + (c == '\r') || + (c == '\f')) + return(TRUE); + else + return(FALSE); + } /* END is_whitespace */ + +int find_range_in_cpuset_string( + + std::string &source, + std::string &output) + + { + output = global_string; + return(0); + } + +void translate_range_string_to_vector( + + const char *range_string, + std::vector &indices) + + { + char *str = strdup(range_string); + char *ptr = str; + int prev; + int curr; + + while (*ptr != '\0') + { + prev = strtol(ptr, &ptr, 10); + + if (*ptr == '-') + { + ptr++; + curr = strtol(ptr, &ptr, 10); + + while (prev <= curr) + { + indices.push_back(prev); + + prev++; + } + + if ((*ptr == ',') || + (is_whitespace(*ptr))) + ptr++; + } + else + { + indices.push_back(prev); + + if ((*ptr == ',') || + (is_whitespace(*ptr))) + ptr++; + } + } + + free(str); + } /* END translate_range_string_to_vector() */ + +bool have_incompatible_dash_l_resource( + + pbs_attribute *pattr) + + { + return(false); + } + +unsigned int complete_req::get_num_reqs() + { + return(1); + } + +req &complete_req::get_req(int i) + { + static req r; + + return(r); + } + +int req::get_gpus() const + + { + return(0); + } + +req::req() {} +req::req(const req &other) {} +req &req::operator =(const req &other) + { + return(*this); + } +std::string req::get_gpu_mode() const + + { + return(this->gpu_mode); + } + diff --git a/src/test/nvidia/test_uut.c b/src/test/nvidia/test_uut.c index b979031dd8..9d39ddfee8 100644 --- a/src/test/nvidia/test_uut.c +++ b/src/test/nvidia/test_uut.c @@ -9,6 +9,7 @@ #include "pbs_error.h" #include "pbs_job.h" +#include "req.hpp" int MOMNvidiaDriverVersion = 0; int LOGLEVEL = 0; @@ -18,9 +19,14 @@ int time_now = 0; char mom_host[PBS_MAXHOSTNAME + 1]; extern nvmlReturn_t global_nvmlDeviceGetFanSpeed_rc; +extern std::map gpu_minor_to_gpu_index; +extern nvmlDevice_t global_device; +extern unsigned int global_device_minor_number; int get_nvml_version(); void generate_server_gpustatus_nvml(std::vector &gpu_status); +int build_gpu_minor_to_gpu_index_map(unsigned int); +int get_gpu_handle_by_minor(unsigned int, nvmlDevice_t*); START_TEST(test_get_nvml_version) { @@ -52,6 +58,23 @@ START_TEST(test_generate_server_gpustatus_nvml) } END_TEST +START_TEST(test_build_gpu_minor_to_gpu_index_map) + { + nvmlDevice_t *handle; + + global_device_minor_number = 0; + build_gpu_minor_to_gpu_index_map(2); + fail_unless(global_device_minor_number == 3); + fail_unless(gpu_minor_to_gpu_index[0] == 0); + fail_unless(gpu_minor_to_gpu_index[1] == 1); + + get_gpu_handle_by_minor(0, handle); + fail_unless(handle == &global_device); + get_gpu_handle_by_minor(1, handle); + fail_unless(handle == &global_device); + } +END_TEST + START_TEST(test_two) { } @@ -72,6 +95,10 @@ Suite *nvidia_suite(void) tcase_add_test(tc_core, test_generate_server_gpustatus_nvml); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_build_gpu_minor_to_gpu_index_map"); + tcase_add_test(tc_core, test_build_gpu_minor_to_gpu_index_map); + suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_two"); tcase_add_test(tc_core, test_two); suite_add_tcase(s, tc_core); diff --git a/src/test/trq_cgroups/scaffolding.c b/src/test/trq_cgroups/scaffolding.c index ff657f536e..5191dc5e78 100644 --- a/src/test/trq_cgroups/scaffolding.c +++ b/src/test/trq_cgroups/scaffolding.c @@ -95,7 +95,7 @@ int rmdir_ext(const char *dirname, int retry_limit) void get_device_indices( const char *device_str, - std::vector &device_indices, + std::vector &device_indices, const char *suffix) {} From 5d62826b3bafb9a001a0699c496149200b9dfcd2 Mon Sep 17 00:00:00 2001 From: Scott Jackson Date: Thu, 25 May 2017 13:06:02 -0600 Subject: [PATCH 268/312] Remove duplicate entries in spec file --- buildutils/torque.adaptive.spec.el6 | 2 -- 1 file changed, 2 deletions(-) diff --git a/buildutils/torque.adaptive.spec.el6 b/buildutils/torque.adaptive.spec.el6 index 012f481381..2c369a54c2 100644 --- a/buildutils/torque.adaptive.spec.el6 +++ b/buildutils/torque.adaptive.spec.el6 @@ -677,8 +677,6 @@ TIMESTAMP="`date +%%Y.%%m.%%d_%%H.%%M.%%S`" %{pre_add_back_up_file %{_bindir}/printjob %{client_sub}-${TIMESTAMP}} %{pre_add_back_up_file %{_bindir}/printserverdb %{client_sub}-${TIMESTAMP}} %{pre_add_back_up_file %{_bindir}/printtracking %{client_sub}-${TIMESTAMP}} -%{pre_add_back_up_file %{_bindir}/printtracking %{client_sub}-${TIMESTAMP}} -%{pre_add_back_up_file %{_bindir}/printtracking %{client_sub}-${TIMESTAMP}} for file in %{_bindir}/q* do %{pre_add_back_up_file ${file} %{client_sub}-${TIMESTAMP}} From 113506786e88fcf9d2b8905a24a971c635b5f858 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 29 Jun 2017 14:12:12 -0600 Subject: [PATCH 269/312] TRQ-4000 - disallow cgroup and geometry request switches concurrently when using configure --- configure.ac | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure.ac b/configure.ac index 848893cf5d..8b2370eed0 100644 --- a/configure.ac +++ b/configure.ac @@ -982,6 +982,9 @@ build_l26_cpuset=no cpuset=0 AC_MSG_CHECKING([whether to enable cpusets]) if test "x$enable_cpuset" = "xyes" ; then + if test "${build_linux_cgroups}" = "yes" ; then + AC_MSG_ERROR([cpuset and cgroups are mutually exclusive options]) + fi AC_MSG_RESULT([yes]) build_l26_cpuset=yes cpuset=1 @@ -1005,6 +1008,9 @@ AC_ARG_ENABLE(geometry_requests, [ if test "x$GCC" = "xyes" ;then AC_MSG_CHECKING([whether to allow geometry requests]) if test "${enable_geometry_requests}" = "yes" ; then + if test "${build_linux_cgroups}" = "yes" ; then + AC_MSG_ERROR([geometry-requests and cgroups are mutually exclusive options]) + fi AC_MSG_RESULT([yes]) build_l26_cpuset=yes AC_DEFINE(PENABLE_LINUX26_CPUSETS, 1, [Define to enable Linux 2.6 cpusets]) From e9cfbc7aab96ad52c504b13eff17ff688f308028 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 10 Jul 2017 18:23:36 -0600 Subject: [PATCH 270/312] TRQ-4068 - avoid post job copy when -k specified --- src/resmom/requests.c | 9 +++++++++ src/test/requests/test_uut.c | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/resmom/requests.c b/src/resmom/requests.c index 92de85a647..a22e01a37a 100644 --- a/src/resmom/requests.c +++ b/src/resmom/requests.c @@ -4397,6 +4397,15 @@ batch_request *get_std_file_info( copy_stdout = false; } + // if files are to be kept, don't copy since they are already at their destination + if (pjob->ji_wattr[JOB_ATR_keep].at_val.at_str != NULL) + { + if (strstr(pjob->ji_wattr[JOB_ATR_keep].at_val.at_str, "o")) + copy_stdout = false; + if (strstr(pjob->ji_wattr[JOB_ATR_keep].at_val.at_str, "e")) + copy_stderr = false; + } + preq = initialize_stageout_request(pjob); struct rq_cpyfile *pcf = &preq->rq_ind.rq_cpyfile; diff --git a/src/test/requests/test_uut.c b/src/test/requests/test_uut.c index 7ac0eb8767..ec544a5f86 100644 --- a/src/test/requests/test_uut.c +++ b/src/test/requests/test_uut.c @@ -3,12 +3,15 @@ #include #include +#include "pbs_config.h" +#include "batch_request.h" #include "pbs_job.h" #include "pbs_error.h" #include "test_requests.h" void string_replchar(const char*, char, char); void determine_spooldir(std::string &spooldir, job *pjob); +batch_request *get_std_file_info(job*); extern char *TNoSpoolDirList[]; extern char *path_spool; @@ -64,6 +67,24 @@ START_TEST(test_string_replchar) } END_TEST +START_TEST(test_get_std_file_info) + { + job myjob; + batch_request *brp; + + memset(&myjob, 0, sizeof(job)); + + // both stdout and stderr are to be kept so don't copy + myjob.ji_wattr[JOB_ATR_keep].at_val.at_str = strdup("eo"); + + brp = get_std_file_info(&myjob); + fail_unless(brp != NULL); + + // now check for empty list (expect no copy requests) + fail_unless(brp->rq_ind.rq_cpyfile.rq_pair.ll_next == brp->rq_ind.rq_cpyfile.rq_pair.ll_prior); + } +END_TEST + Suite *requests_suite(void) { @@ -76,6 +97,10 @@ Suite *requests_suite(void) tcase_add_test(tc_core, test_determine_spooldir); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_get_std_file_info"); + tcase_add_test(tc_core, test_get_std_file_info); + suite_add_tcase(s, tc_core); + return s; } From c615ccf61fdc449709146ec47c212315a1719a5f Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 12 Jul 2017 15:55:10 -0600 Subject: [PATCH 271/312] TRQ-4052 - Reword --enable-nvidia-gpus configure option --- configure.ac | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 8b2370eed0..82934fd74f 100644 --- a/configure.ac +++ b/configure.ac @@ -1198,7 +1198,8 @@ AC_ARG_ENABLE(nvidia-gpus, [ --enable-nvidia-gpus enable Nvidia gpu support Nvidia gpu support requires the use of the Nvidia Management Library (NVML) When using --enable-nvidia-gpus, you must also specify --with-nvml-lib=DIR and - --with-nvml-include=DIR. hwloc 1.9 or later is also required. + --with-nvml-include=DIR. + hwloc 1.9 or later is also required if implementing cgroups along with GPU support. See --with-hwloc-path. ], enable_nvidia=$enableval,enable_nvidia=no) From 8811f45b12974e5991532449f762f56ac17a24c6 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 20 Jul 2017 12:53:59 -0600 Subject: [PATCH 272/312] TRQ-4085 - take last qsub option --- src/lib/Libutils/u_hash_map_structs.c | 5 +++-- src/test/u_hash_map_structs/test_uut.c | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/lib/Libutils/u_hash_map_structs.c b/src/lib/Libutils/u_hash_map_structs.c index 9e3c37f91e..5d6f6dc35f 100644 --- a/src/lib/Libutils/u_hash_map_structs.c +++ b/src/lib/Libutils/u_hash_map_structs.c @@ -121,7 +121,7 @@ void hash_priority_add_or_exit( job_data_container *head, const char *name, const char *value, - int var_type) + int var_type) // lower values have higher priority (see "_DATA" suffixed macros in pbs_contants.h) { bool should_add = true; @@ -130,7 +130,8 @@ void hash_priority_add_or_exit( if (old_item != NULL) { // Only call insert if we have priority over the old item - if (var_type >= old_item->var_type) + // Note that items with the same priority will be added + if (var_type > old_item->var_type) should_add = false; } diff --git a/src/test/u_hash_map_structs/test_uut.c b/src/test/u_hash_map_structs/test_uut.c index 6f11c84768..873877dc16 100644 --- a/src/test/u_hash_map_structs/test_uut.c +++ b/src/test/u_hash_map_structs/test_uut.c @@ -34,12 +34,18 @@ START_TEST(test_hash_priority_add_or_exit) hash_priority_add_or_exit(&map, "planet", "nalthis", SCRIPT_DATA); job_data *stored; - fail_unless(hash_find(&map, "planet", &stored) == 1); + fail_unless(hash_find(&map, "planet", &stored) == TRUE); fail_unless(stored->value == "sel"); // command line is higher priority than script hash_priority_add_or_exit(&map, "planet", "roshar", ENV_DATA); - fail_unless(hash_find(&map, "planet", &stored) == 1); + fail_unless(hash_find(&map, "planet", &stored) == TRUE); fail_unless(stored->value == "roshar"); // environment data is higher priority than command line + + // insert a duplicate item + char *planet_name = strdup("mars"); + hash_priority_add_or_exit(&map, "planet", planet_name, ENV_DATA); + fail_unless(hash_find(&map, "planet", &stored) == TRUE); + fail_unless(stored->value == planet_name); // should retrieve lastest entry added } END_TEST From e51e53c8e7be880fa6b2f647a9fc605d8204b1bf Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 14 Aug 2017 10:26:32 -0600 Subject: [PATCH 273/312] TRQ-4065 - add logic to exit array processing loop --- src/server/pbsd_init.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/server/pbsd_init.c b/src/server/pbsd_init.c index 26a0e8302f..ec595741c9 100644 --- a/src/server/pbsd_init.c +++ b/src/server/pbsd_init.c @@ -1988,6 +1988,7 @@ int cleanup_recovered_arrays() char arrayid[PBS_MAXSVRJOBID+1]; all_arrays_iterator *iter = NULL; int rc = PBSE_NONE; + char log_buf[LOCAL_LOG_BUF_SIZE]; while ((pa = next_array(&iter)) != NULL) { @@ -2032,6 +2033,10 @@ int cleanup_recovered_arrays() if (job_template_exists == FALSE) { + snprintf(log_buf, sizeof(log_buf), "Cleaning up job array %s of size %d that could not be built during ini t.", + arrayid, pa->ai_qs.array_size); + log_event(PBSEVENT_DEBUG | PBSEVENT_FORCE, PBS_EVENTCLASS_JOB, __func__, log_buf); + for (int i = 0; i < pa->ai_qs.array_size; i++) { if (pa->job_ids[i] != NULL) @@ -2042,6 +2047,10 @@ int cleanup_recovered_arrays() svr_job_purge(pjob); pa = get_array(arrayid); + if (pa == NULL) + { + break; + } } } } @@ -2049,6 +2058,13 @@ int cleanup_recovered_arrays() std::string array_id(pa->ai_qs.parent_id); unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); array_delete(array_id.c_str()); + if (pa != NULL) + { + std::string array_id(pa->ai_qs.parent_id); + unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); + array_delete(array_id.c_str()); + } + continue; } else From 9e22c0b84ee5b62384ea4039e34b95a02344f8ae Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 30 Aug 2017 11:31:47 -0600 Subject: [PATCH 274/312] TRQ-4099 Modify Socket::spread_place update --- src/lib/Libutils/numa_socket.cpp | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/lib/Libutils/numa_socket.cpp b/src/lib/Libutils/numa_socket.cpp index 22279b7420..cd34515c56 100644 --- a/src/lib/Libutils/numa_socket.cpp +++ b/src/lib/Libutils/numa_socket.cpp @@ -655,23 +655,26 @@ bool Socket::spread_place( numa_nodes_required = this->chips.size(); this->socket_exclusive = true; } - + + // In order to evenly spread out resources across nodes, we divide number of nodes + // by resources required and are left with number of resources per node plus a + // remainder. As we iterate over the nodes and allocate resources, we take one + // resource from the remainder every step until there is nothing left in the remainder. + allocation numa_remainder(remaining); + numa_remainder.adjust_for_spread(numa_nodes_required, true); + for (int c = 0; c < numa_nodes_required; c++) { allocation per_numa(remaining); - allocation numa_remainder(remaining); - - if (numa_nodes_required == 1) - numa_remainder.clear(); - else - { - per_numa.adjust_for_spread(numa_nodes_required, false); - numa_remainder.adjust_for_spread(numa_nodes_required, true); - } - - for (unsigned int i = 0; i < this->chips.size(); i++) + per_numa.adjust_for_spread(numa_nodes_required, false); + + per_numa.adjust_for_remainder(numa_remainder); + + // This inner loop is for the case where we are not reserving the whole + // socket, but a single chip. In the case of a single chip, we iterate + // over all chips to see if there is a chip which is completely free. + for (unsigned int i = c; i < this->chips.size(); i++) { - per_numa.adjust_for_remainder(numa_remainder); if (this->chips[i].spread_place(r, task_alloc, per_numa, remainder)) { placed = true; From c5e1199b0851bff73b53906beec6c571a29740b6 Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Thu, 25 May 2017 16:33:14 -0600 Subject: [PATCH 275/312] TRQ-1853. Send success code to orphaned nodes to reduce update freq --- src/server/receive_mom_communication.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server/receive_mom_communication.c b/src/server/receive_mom_communication.c index 69e2cc2e41..954d698ae0 100644 --- a/src/server/receive_mom_communication.c +++ b/src/server/receive_mom_communication.c @@ -432,7 +432,10 @@ void *svr_is_request( { log_err(-1, __func__, log_buf); } - + + // Reduce "spamming" from nodes not authorized + write_tcp_reply(chan, IS_PROTOCOL, IS_PROTOCOL_VER, IS_STATUS, DIS_SUCCESS); + close_conn(chan->sock, FALSE); DIS_tcp_cleanup(chan); return(NULL); From 7f001fa7b41e99b755c139b3e5be3e297e19e07e Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Tue, 30 May 2017 12:27:21 -0600 Subject: [PATCH 276/312] TRQ-2994. Add node_check_on_job_start and node_check_on_job_end --- src/include/mom_config.h | 2 ++ src/include/utils.h | 3 +++ src/resmom/mom_main.c | 47 +++++++++++++++++++++++++++++++++++++++ src/resmom/parse_config.c | 44 ++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) diff --git a/src/include/mom_config.h b/src/include/mom_config.h index 6d44a3d9e5..dbcf32e3c1 100644 --- a/src/include/mom_config.h +++ b/src/include/mom_config.h @@ -197,6 +197,8 @@ unsigned long setenablemomrestart(const char *value); unsigned long setrcpcmd(const char *value); unsigned long setjobdirectorysticky(const char *value); unsigned long setcudavisibledevices(const char *value); +unsigned long setnodecheckonjobstart(const char *value); +unsigned long setnodecheckonjobend(const char *value); #ifdef PENABLE_LINUX26_CPUSETS unsigned long setmempressthr(const char *); diff --git a/src/include/utils.h b/src/include/utils.h index 1aaafe6fca..3b2aa80314 100644 --- a/src/include/utils.h +++ b/src/include/utils.h @@ -209,5 +209,8 @@ int rmdir_ext(const char *dir, int retry_limit = 20); int unlink_ext(const char *filename, int retry_limit = 20); int mkdir_wrapper(const char *pathname, mode_t mode); +/* from parse_config.c */ +int setbool(const char *value); + #endif /* END #ifndef UTILS_H */ diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index 56dee522d4..bf24bb02a1 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -2499,6 +2499,45 @@ void set_report_mom_cuda_visible_devices( +void set_report_node_check_on_job_start( + + std::stringstream &output, + char *curr) + + { + int enable; + + if ((*curr == '=') && ((*curr) + 1 != '\0')) + { + if ((enable = setbool(curr + 1)) != -1) + PBSNodeCheckProlog = enable; + } + + output << "node_check_on_job_start=" << PBSNodeCheckProlog; + + } /* END set_node_check_on_job_start() */ + + + +void set_report_node_check_on_job_end( + + std::stringstream &output, + char *curr) + + { + int enable; + + if ((*curr == '=') && ((*curr) + 1 != '\0')) + { + if ((enable = setbool(curr + 1)) != -1) + PBSNodeCheckEpilog = enable; + } + + output << "node_check_on_job_end=" << PBSNodeCheckEpilog; + } /* END set_node_check_on_job_end() */ + + + void set_report_rcpcmd( std::stringstream &output, @@ -2761,6 +2800,14 @@ int process_rm_cmd_request( { set_report_mom_cuda_visible_devices(output, curr); } + else if (!strncasecmp(name, "node_check_on_job_start", strlen("node_check_on_job_start"))) + { + set_report_node_check_on_job_start(output, curr); + } + else if (!strncasecmp(name, "node_check_on_job_end", strlen("node_check_on_job_end"))) + { + set_report_node_check_on_job_end(output, curr); + } else { report_other_configured_attribute(output, name, curr, cp, restrictrm); diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index 4743741a34..69bacc9153 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -313,6 +313,8 @@ unsigned long set_presetup_prologue(const char *); unsigned long set_max_physical_memory(const char *); unsigned long set_max_swap_memory(const char *); unsigned long set_get_cray_taskstats(const char *); +unsigned long set_node_check_on_job_start(const char *); +unsigned long set_node_check_on_job_end(const char *); struct specials special[] = { { "force_overwrite", setforceoverwrite}, @@ -401,6 +403,8 @@ struct specials special[] = { { "max_physical_memory", set_max_physical_memory}, { "max_swap_memory", set_max_swap_memory}, { "get_cray_taskstats", set_get_cray_taskstats}, + { "node_check_on_job_start", set_node_check_on_job_start }, + { "node_check_on_job_end", set_node_check_on_job_end }, { NULL, NULL } }; @@ -819,6 +823,46 @@ unsigned long setnodecheckinterval( +unsigned long set_node_check_on_job_start( + + const char *value) + + { + int enable; + + log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, __func__, value); + + if ((enable = setbool(value)) != -1) + { + PBSNodeCheckProlog = enable; + return(1); + } + + return(0); + } /* END set_node_check_on_job_start() */ + + + +unsigned long set_node_check_on_job_end( + + const char *value) + + { + int enable; + + log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, __func__, value); + + if ((enable = setbool(value)) != -1) + { + PBSNodeCheckEpilog = enable; + return(1); + } + + return(0); + } /* END set_node_check_on_job_end() */ + + + unsigned long settimeout( const char *value) From 0bd8f291fc8b37e290865c9b10664a27e5c9f6a8 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 13 Sep 2017 16:37:49 -0600 Subject: [PATCH 277/312] Fix unit tests. --- src/test/pbsD_submit/scaffolding.c | 6 ++++++ src/test/pbsD_submit_hash/scaffolding.c | 6 ++++++ src/test/trq_cgroups/scaffolding.c | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/src/test/pbsD_submit/scaffolding.c b/src/test/pbsD_submit/scaffolding.c index 74b793db1a..99cc856c4d 100644 --- a/src/test/pbsD_submit/scaffolding.c +++ b/src/test/pbsD_submit/scaffolding.c @@ -18,6 +18,12 @@ int PBSD_commit(int connect, char *jobid) exit(1); } +int PBSD_commit2(int connect, char *jobid) + { + fprintf(stderr, "The call to PBSD_commit needs to be mocked!!\n"); + exit(1); + } + int PBSD_rdytocmt(int connect, char *jobid) { fprintf(stderr, "The call to PBSD_rdytocmt needs to be mocked!!\n"); diff --git a/src/test/pbsD_submit_hash/scaffolding.c b/src/test/pbsD_submit_hash/scaffolding.c index 8fe49af129..6f03541434 100644 --- a/src/test/pbsD_submit_hash/scaffolding.c +++ b/src/test/pbsD_submit_hash/scaffolding.c @@ -21,6 +21,12 @@ int PBSD_commit(int connect, char *jobid) exit(1); } +int PBSD_commit2(int connect, char *jobid) + { + fprintf(stderr, "The call to PBSD_commit needs to be mocked!!\n"); + exit(1); + } + int PBSD_rdytocmt(int connect, char *jobid) { fprintf(stderr, "The call to PBSD_rdytocmt needs to be mocked!!\n"); diff --git a/src/test/trq_cgroups/scaffolding.c b/src/test/trq_cgroups/scaffolding.c index 5191dc5e78..8308bb93eb 100644 --- a/src/test/trq_cgroups/scaffolding.c +++ b/src/test/trq_cgroups/scaffolding.c @@ -160,3 +160,11 @@ void log_event( global_trq_cg_signal_tasks_ok = true; } } + +int translate_range_string_to_vector( + + const char *range_string, + std::vector &indices) +{ +return(0); +} From 859574194cd973dcc8cfe1d1b9448033fb869a49 Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Tue, 23 May 2017 15:09:09 -0600 Subject: [PATCH 278/312] TRQ-3512. Maintain $pbsclient value over MOM hierarchy updates. --- src/include/mom_config.h | 1 + src/resmom/mom_server.c | 5 +++++ src/resmom/parse_config.c | 12 ++++++++---- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/include/mom_config.h b/src/include/mom_config.h index dbcf32e3c1..ace8931004 100644 --- a/src/include/mom_config.h +++ b/src/include/mom_config.h @@ -178,6 +178,7 @@ extern std::string presetup_prologue; extern unsigned long max_memory; extern unsigned long max_swap; extern bool get_cray_taskstats; +extern u_long pbsclient; struct specials { diff --git a/src/resmom/mom_server.c b/src/resmom/mom_server.c index f93e01f2ab..d151f43bc2 100644 --- a/src/resmom/mom_server.c +++ b/src/resmom/mom_server.c @@ -2362,6 +2362,11 @@ void reset_okclients() } } + if (pbsclient != 0) + { + auth_hosts.add_authorized_address(pbsclient, 0, ""); + } + // add localhost auth_hosts.add_authorized_address(localaddr, 0, ""); diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index 69bacc9153..8c4bb8585d 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -200,6 +200,7 @@ std::string presetup_prologue; unsigned long max_memory = 0; unsigned long max_swap = 0; bool get_cray_taskstats = false; +u_long pbsclient; @@ -1558,19 +1559,22 @@ u_long setpbsclient( { /* FAILURE */ - return(1); + return(0); } rc = addclient(value); - if (rc != 0) + // addclient() returns the address on success, 0 on failure + if (rc == 0) { /* FAILURE */ - return(1); + return(0); } - return(0); + pbsclient = rc; + + return(1); } /* END setpbsclient() */ From 4aee4fe22b8b95b0f7289472577ea37c7b522608 Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Tue, 16 May 2017 17:46:39 -0600 Subject: [PATCH 279/312] TRQ-3957. Do not run epilogues unless prologues have run. --- src/include/pbs_job.h | 1 + src/resmom/catch_child.c | 6 ++++++ src/resmom/start_exec.c | 4 ++++ src/test/catch_child/scaffolding.c | 3 +++ src/test/catch_child/test_uut.c | 19 +++++++++++++++++++ 5 files changed, 33 insertions(+) diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 709292e638..48363548c1 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -1156,6 +1156,7 @@ class send_job_request //#define JOB_SVFLG_RescAssn 0x2000 // Not used on the mom //#define JOB_SVFLG_CHECKPOINT_COPIED 0x4000 // Not used on the mom #define JOB_SVFLG_INTERMEDIATE_MOM 0x8000 // This is for job_radix. I am an intermediate mom +#define JOB_SVFLG_PROLOGUES_RAN 0x10000 // job has run prologues #endif diff --git a/src/resmom/catch_child.c b/src/resmom/catch_child.c index 195111e36c..59820ff1bd 100644 --- a/src/resmom/catch_child.c +++ b/src/resmom/catch_child.c @@ -1070,6 +1070,12 @@ int run_epilogues( int rc; + if (!(pjob->get_svrflags() & JOB_SVFLG_PROLOGUES_RAN)) + { + log_err(-1, __func__, "prologues were not run; skipping epilogues"); + return(PBSE_NONE); + } + if ((pjob->ji_wattr[JOB_ATR_interactive].at_flags & ATR_VFLAG_SET) && pjob->ji_wattr[JOB_ATR_interactive].at_val.at_long) { diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index a286594831..6d7f5f1644 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -3523,6 +3523,8 @@ void handle_prologs( } } + pjob->set_svrflags(pjob->get_svrflags() | JOB_SVFLG_PROLOGUES_RAN); + } /* END handle_prologs() */ @@ -7392,6 +7394,8 @@ int start_exec( execute_presetup_prologue(pjob); } + pjob->set_svrflags(pjob->get_svrflags() | JOB_SVFLG_PROLOGUES_RAN); + /* start_exec only executed on mother superior */ pjob->ji_nodeid = 0; /* I'm MS */ nodenum = pjob->ji_numnodes; diff --git a/src/test/catch_child/scaffolding.c b/src/test/catch_child/scaffolding.c index 92af599383..7db1077854 100644 --- a/src/test/catch_child/scaffolding.c +++ b/src/test/catch_child/scaffolding.c @@ -25,6 +25,7 @@ int server_down; int called_open_socket = 0; int called_fork_me = 0; +bool called_epilogue = false; bool check_rur = true; char mom_alias[PBS_MAXHOSTNAME + 1]; @@ -1162,6 +1163,8 @@ char *get_local_script_path(job *pjob, char *base) int run_pelog(int which, char *specpelog, job *pjog, int pe_io_type, int deletejob) { + called_epilogue = true; + return 1; } diff --git a/src/test/catch_child/test_uut.c b/src/test/catch_child/test_uut.c index 4e967c827a..6f6a9cac4e 100644 --- a/src/test/catch_child/test_uut.c +++ b/src/test/catch_child/test_uut.c @@ -13,6 +13,7 @@ bool non_mother_superior_cleanup(job *pjob); bool mother_superior_cleanup(job *pjob, int limit, int *found); void *obit_reply(void *new_sock); hnodent *get_node(job *pjob, tm_node_id nodeid); +int run_epilogues(mom_job *pjob, int i_am_ms, int deletejob); extern int termin_child; extern int server_down; @@ -21,6 +22,7 @@ extern int DIS_reply_read_count; extern int tc; extern int called_open_socket; extern int called_fork_me; +extern bool called_epilogue; extern bool eintr_test; extern std::vector exiting_job_list; @@ -200,6 +202,22 @@ START_TEST(test_mother_superior_cleanup) } END_TEST +START_TEST(test_epilogues_run_without_prologues) + { + mom_job *pjob = new mom_job(); + + pjob->set_svrflags(JOB_SVFLG_PROLOGUES_RAN); + called_epilogue = false; + run_epilogues(pjob, 0, 0); + ck_assert(called_epilogue); + + pjob->set_svrflags(0x0); + called_epilogue = false; + run_epilogues(pjob, 0, 0); + ck_assert(!called_epilogue); + } +END_TEST + Suite *catch_child_suite(void) { Suite *s = suite_create("catch_child methods"); @@ -210,6 +228,7 @@ Suite *catch_child_suite(void) tcase_add_test(tc_core, test_jobs_main_process); tcase_add_test(tc_core, test_non_mother_superior_cleanup); tcase_add_test(tc_core, test_mother_superior_cleanup); + tcase_add_test(tc_core, test_epilogues_run_without_prologues); s = suite_create("get_node_tests"); tcase_add_test(tc_core, test_get_node_1); From 4b507610eccd8d006eb93b332950f335cd5bfe00 Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Thu, 25 May 2017 17:01:01 -0600 Subject: [PATCH 280/312] TRQ-4054. Add '--xml' flag as alias for '-x' for supported commands --- src/cmds/pbsnodes.c | 7 +++++++ src/cmds/qstat.c | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/cmds/pbsnodes.c b/src/cmds/pbsnodes.c index 633351454b..7fa23acac6 100644 --- a/src/cmds/pbsnodes.c +++ b/src/cmds/pbsnodes.c @@ -920,6 +920,13 @@ int main( { TShowAbout_exit(); } + else if ((optarg != NULL) && !strcmp(optarg, "xml")) + { + flag = ALLI; + DisplayXML = TRUE; + + break; + } errflg = 1; diff --git a/src/cmds/qstat.c b/src/cmds/qstat.c index a4b19a810c..e6f819d5e3 100644 --- a/src/cmds/qstat.c +++ b/src/cmds/qstat.c @@ -2828,6 +2828,16 @@ int process_commandline_opts( TShowAbout_exit(); } + if ((optarg != NULL) && !strcmp(optarg, "xml")) + { + DisplayXML = true; + + /* We want to return all attributes */ + attrib = NULL; + + break; + } + /* unexpected '--' option received */ errflg = 1; From d323329afea7c47ee0bfcfb42eeede350d918736 Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Thu, 14 Sep 2017 15:57:36 -0600 Subject: [PATCH 281/312] TRQ-3957. Revision to earlier commit. --- src/resmom/catch_child.c | 2 +- src/resmom/start_exec.c | 4 ++-- src/test/catch_child/test_uut.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/resmom/catch_child.c b/src/resmom/catch_child.c index 59820ff1bd..72ed7857e3 100644 --- a/src/resmom/catch_child.c +++ b/src/resmom/catch_child.c @@ -1070,7 +1070,7 @@ int run_epilogues( int rc; - if (!(pjob->get_svrflags() & JOB_SVFLG_PROLOGUES_RAN)) + if (!(pjob->ji_qs.ji_svrflags & JOB_SVFLG_PROLOGUES_RAN)) { log_err(-1, __func__, "prologues were not run; skipping epilogues"); return(PBSE_NONE); diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index 6d7f5f1644..fe4a3e99cd 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -3523,7 +3523,7 @@ void handle_prologs( } } - pjob->set_svrflags(pjob->get_svrflags() | JOB_SVFLG_PROLOGUES_RAN); + pjob->ji_qs.ji_svrflags |= JOB_SVFLG_PROLOGUES_RAN; } /* END handle_prologs() */ @@ -7394,7 +7394,7 @@ int start_exec( execute_presetup_prologue(pjob); } - pjob->set_svrflags(pjob->get_svrflags() | JOB_SVFLG_PROLOGUES_RAN); + pjob->ji_qs.ji_svrflags |= JOB_SVFLG_PROLOGUES_RAN; /* start_exec only executed on mother superior */ pjob->ji_nodeid = 0; /* I'm MS */ diff --git a/src/test/catch_child/test_uut.c b/src/test/catch_child/test_uut.c index 6f6a9cac4e..4478853644 100644 --- a/src/test/catch_child/test_uut.c +++ b/src/test/catch_child/test_uut.c @@ -206,12 +206,12 @@ START_TEST(test_epilogues_run_without_prologues) { mom_job *pjob = new mom_job(); - pjob->set_svrflags(JOB_SVFLG_PROLOGUES_RAN); + pjob->ji_qs.ji_svrflags |= JOB_SVFLG_PROLOGUES_RAN; called_epilogue = false; run_epilogues(pjob, 0, 0); ck_assert(called_epilogue); - pjob->set_svrflags(0x0); + pjob->ji_qs.ji_svrflags = 0x0; called_epilogue = false; run_epilogues(pjob, 0, 0); ck_assert(!called_epilogue); From f8d2afa1fb38194d38a28fa38bcd21946945ee8b Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 15 Sep 2017 11:04:43 -0600 Subject: [PATCH 282/312] Fix unit tests. --- src/test/catch_child/test_uut.c | 4 ++-- src/test/mom_main/test_uut.c | 2 ++ src/test/mom_server/test_uut.c | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/catch_child/test_uut.c b/src/test/catch_child/test_uut.c index 4478853644..d5e9341e55 100644 --- a/src/test/catch_child/test_uut.c +++ b/src/test/catch_child/test_uut.c @@ -13,7 +13,7 @@ bool non_mother_superior_cleanup(job *pjob); bool mother_superior_cleanup(job *pjob, int limit, int *found); void *obit_reply(void *new_sock); hnodent *get_node(job *pjob, tm_node_id nodeid); -int run_epilogues(mom_job *pjob, int i_am_ms, int deletejob); +int run_epilogues(job *pjob, int i_am_ms, int deletejob); extern int termin_child; extern int server_down; @@ -204,7 +204,7 @@ END_TEST START_TEST(test_epilogues_run_without_prologues) { - mom_job *pjob = new mom_job(); + job *pjob = (job *)calloc(1, sizeof(job)); pjob->ji_qs.ji_svrflags |= JOB_SVFLG_PROLOGUES_RAN; called_epilogue = false; diff --git a/src/test/mom_main/test_uut.c b/src/test/mom_main/test_uut.c index e7a504f065..f751b222f9 100644 --- a/src/test/mom_main/test_uut.c +++ b/src/test/mom_main/test_uut.c @@ -22,6 +22,8 @@ extern bool ForceServerUpdate; extern int MOMCudaVisibleDevices; extern bool daemonize_mom; +int PBSNodeCheckProlog; +int PBSNodeCheckEpilog; time_t pbs_tcp_timeout; unsigned long setcudavisibledevices(const char *); void set_report_mom_cuda_visible_devices(std::stringstream &output, char *curr); diff --git a/src/test/mom_server/test_uut.c b/src/test/mom_server/test_uut.c index bdd08001ab..fe2d2c1b2f 100644 --- a/src/test/mom_server/test_uut.c +++ b/src/test/mom_server/test_uut.c @@ -20,6 +20,7 @@ int mom_server_update_stat(mom_server *pms, std::vector &strings); char PBSNodeMsgBuf[MAXLINE]; char PBSNodeCheckPath[MAXLINE]; int PBSNodeCheckInterval = 2; +u_long pbsclient; extern bool no_error; extern bool no_event; From f1a604f5016482d83f62278fd68cbc7f797977ba Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Fri, 15 Sep 2017 14:48:40 -0600 Subject: [PATCH 283/312] TRQ-3957. Run epilogue on sister nodes properly. --- src/resmom/mom_comm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index 64da90e152..7b34686076 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -8880,6 +8880,8 @@ int run_prologue_scripts( ret = PBSE_NONE; done: + pjob->ji_qs.ji_svrflags |= JOB_SVFLG_PROLOGUES_RAN; + return(ret); } /* END run_prologue_scripts() */ From 27156b6aae5271015e92c1a1395ca5ce019aea9f Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Thu, 25 May 2017 17:44:18 -0600 Subject: [PATCH 284/312] TRQ-3746. Make sure pbs_server reads the pbs_environment file --- src/resmom/mom_main.c | 24 ++++++++++++------------ src/server/pbsd_init.c | 13 ------------- src/server/pbsd_main.c | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index bf24bb02a1..9f07badcbe 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -4908,18 +4908,6 @@ int setup_program_environment(void) umask(022); - if (getenv("PBSLOGLEVEL") != NULL) - { - LOGLEVEL = (int)strtol(getenv("PBSLOGLEVEL"), NULL, 0); - } - - if (getenv("PBSDEBUG") != NULL) - { - DEBUGMODE = 1; - - daemonize_mom = false; - } - memset(TMOMStartInfo, 0, sizeof(pjobexec_t)*TMAX_JE); /* modify program environment */ @@ -4937,6 +4925,18 @@ int setup_program_environment(void) exit(1); } + if (getenv("PBSLOGLEVEL") != NULL) + { + LOGLEVEL = (int)strtol(getenv("PBSLOGLEVEL"), NULL, 0); + } + + if (getenv("PBSDEBUG") != NULL) + { + DEBUGMODE = 1; + + daemonize_mom = false; + } + c = getgid(); /* secure suppl. groups */ diff --git a/src/server/pbsd_init.c b/src/server/pbsd_init.c index ec595741c9..50dba27043 100644 --- a/src/server/pbsd_init.c +++ b/src/server/pbsd_init.c @@ -2270,19 +2270,6 @@ int pbsd_init( memset(&hints, 0, sizeof(hints)); hints.ai_flags = AI_CANONNAME; - /* The following is code to reduce security risks */ - if (use_path_home == true) - { - path_pbs_environment = build_path(path_home, "pbs_environment", ""); - if (setup_env(path_pbs_environment) == -1) - { - return(-1); - } - } - else if (setup_env(PBS_ENVIRON) == -1) - { - return(-1); - } gid = getgid(); diff --git a/src/server/pbsd_main.c b/src/server/pbsd_main.c index c6db8b05b0..7d719c723c 100644 --- a/src/server/pbsd_main.c +++ b/src/server/pbsd_main.c @@ -111,6 +111,7 @@ #include "../lib/Liblog/pbs_log.h" #include "../lib/Liblog/log_event.h" #include "../lib/Liblog/chk_file_sec.h" +#include "../lib/Liblog/setup_env.h" #include "lib_ifl.h" #include "server_limits.h" #include "attribute.h" @@ -171,6 +172,7 @@ extern int RPPConfigure(int, int); extern void acct_cleanup(long); void stream_eof(int, u_long, uint16_t, int); extern void scheduler_close(); +extern char *build_path(char *, const char *, const char *); #ifndef MAX_PATH_LEN #define MAX_PATH_LEN 256 @@ -1619,6 +1621,21 @@ int main( memset(&server, 0, sizeof(struct server)); log_init(NULL, NULL); initialize_globals(); + + /* The following is code to reduce security risks */ + if (use_path_home == true) + { + path_pbs_environment = build_path(path_home, "pbs_environment", ""); + if (setup_env(path_pbs_environment) == -1) + { + exit(1); + } + } + else if (setup_env(PBS_ENVIRON) == -1) + { + exit(1); + } + set_globals_from_environment(); /* set standard umask */ From 23266fe781ce41a167a0a2f312693a0ca6fedd0d Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Mon, 3 Jul 2017 15:56:02 -0600 Subject: [PATCH 285/312] TRQ-4072. Add bind() before client-side connect() calls --- configure.ac | 12 ++++++ src/lib/Libifl/tm.c | 26 +++++++++++++ src/lib/Libifl/trq_auth.c | 26 +++++++++++++ src/lib/Libnet/lib_net.h | 1 + src/lib/Libnet/net_common.c | 64 +++++++++++++++++++++++++++++--- src/lib/Libnet/port_forwarding.c | 25 +++++++++++++ src/lib/Libnet/rm.c | 27 ++++++++++++-- src/resmom/start_exec.c | 31 ++++++++++++++++ src/test/net_common/test_uut.c | 26 +++++++++++++ 9 files changed, 229 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index 82934fd74f..b2a31722bd 100644 --- a/configure.ac +++ b/configure.ac @@ -1858,6 +1858,18 @@ esac +dnl Make a bind() call before making a connect() call +AC_MSG_CHECKING([whether to build with bind() calls before connect() calls]) +AC_ARG_ENABLE(bind-outbound-sockets, + [ --enable-bind-outbound-sockets enable build with bind() calls before connect() calls], + BIND_OUTBOUND_SOCKETS=$enableval,BIND_OUTBOUND_SOCKETS=yes) +AC_MSG_RESULT($BIND_OUTBOUND_SOCKETS) +if test "$BIND_OUTBOUND_SOCKETS" = "yes" ; then + AC_DEFINE(BIND_OUTBOUND_SOCKETS, 1, [turns on the compilation of BIND_OUTBOUND_SOCKETS code]) +fi + + + dnl dnl ###################################################################### dnl Scheduler settings diff --git a/src/lib/Libifl/tm.c b/src/lib/Libifl/tm.c index a08cf384bb..d12a8d79ce 100644 --- a/src/lib/Libifl/tm.c +++ b/src/lib/Libifl/tm.c @@ -484,6 +484,11 @@ static int localmom(void) struct linger ltime; +#ifdef BIND_OUTBOUND_SOCKETS + struct sockaddr_in local; +#endif + + if (local_conn >= 0) { return(local_conn); /* already have open connection */ @@ -528,6 +533,27 @@ static int localmom(void) #endif +#ifdef BIND_OUTBOUND_SOCKETS + + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + + if (get_local_address(local) != 0) + { + TM_DBPRT(("tm_init: unable to determine local IP address")); + close(sock); + return(-1); + } + + if (bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in)) != 0) + { + TM_DBPRT(("tm_init: unable to bind to local IP address")); + close(sock); + return(-1); + } +#endif + + /* make sure data goes out */ ltime.l_onoff = 1; diff --git a/src/lib/Libifl/trq_auth.c b/src/lib/Libifl/trq_auth.c index 5e0fced2c5..524b14c404 100644 --- a/src/lib/Libifl/trq_auth.c +++ b/src/lib/Libifl/trq_auth.c @@ -331,6 +331,19 @@ int trq_simple_connect( int optval = 1; std::string server(server_name); +#ifdef BIND_OUTBOUND_SOCKETS + struct sockaddr_in local; + + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + if (get_local_address(local) != PBSE_NONE) + { + fprintf(stderr, "could not determine local IP address: %s", strerror(errno)); + return(PBSE_SYSTEM); + } + +#endif + memset(&hints, 0, sizeof(hints)); /* set the hints so we get a STREAM socket */ hints.ai_family = AF_UNSPEC; /* allow for IPv4 or IPv6 */ @@ -365,6 +378,19 @@ int trq_simple_connect( continue; } +#ifdef BIND_OUTBOUND_SOCKETS + + rc = bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in)); + if (rc != 0) + { + fprintf(stderr, "could not bind local socket: %s", strerror(errno)); + rc = PBSE_SYSTEM; + close(sock); + continue; + } + +#endif + rc = connect(sock, addr_info->ai_addr, addr_info->ai_addrlen); if (rc != 0) { diff --git a/src/lib/Libnet/lib_net.h b/src/lib/Libnet/lib_net.h index 6b5929c7ea..4e008ecfb8 100644 --- a/src/lib/Libnet/lib_net.h +++ b/src/lib/Libnet/lib_net.h @@ -52,6 +52,7 @@ int socket_read_str(int socket, char **the_str, long long *str_len); int socket_close(int socket); int pbs_getaddrinfo(const char *pNode,struct addrinfo *pHints,struct addrinfo **ppAddrInfoOut); int connect_to_trqauthd(int *sock); +int get_local_address(struct sockaddr_in &new_sockaddr); /* from file server_core.c */ diff --git a/src/lib/Libnet/net_common.c b/src/lib/Libnet/net_common.c index c6006f61c6..dc4e5702d4 100644 --- a/src/lib/Libnet/net_common.c +++ b/src/lib/Libnet/net_common.c @@ -205,13 +205,22 @@ int socket_get_tcp_priv() int priv_port = 0; int flags; #endif - - memset(&local, 0, sizeof(struct sockaddr_in)); - local.sin_family = AF_INET; - + if ((local_socket = socket_get_tcp()) < 0) return(-1); +#ifdef BIND_OUTBOUND_SOCKETS + if (get_local_address(local) != PBSE_NONE) + { + close(local_socket); + return(-1); + } +#else + memset(&local, 0, sizeof(struct sockaddr_in)); + local.sin_family = AF_INET; +#endif + + #ifndef NOPRIVPORTS /* According to the notes in the previous code: * bindresvport seems to cause connect() failures in some odd corner case @@ -1030,4 +1039,49 @@ int connect_to_trqauthd( return(rc); } - + +/* + * get_local_address() - lookup the local IP address associated with the local, + * public hostname. + * + * This function will cache the IP address in each process to speed up + * frequent lookups. + */ +int get_local_address( + + struct sockaddr_in &new_sockaddr) + + { + static struct addrinfo *local_addr_info = NULL; + struct addrinfo hints; + int rc; + char hostname[PBS_MAXHOSTNAME]; + hostname[PBS_MAXHOSTNAME - 1] = '\0'; + + // If we already have the local address cached, return (copy) that value. + if (local_addr_info != NULL) + { + new_sockaddr = *((sockaddr_in*)local_addr_info->ai_addr); + + return PBSE_NONE; + } + + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + gethostname(hostname, sizeof(hostname)); + + rc = getaddrinfo(hostname, NULL, &hints, &local_addr_info); + if (rc != 0) + { + return(rc); + } + + new_sockaddr = *((sockaddr_in*)local_addr_info->ai_addr); + + return PBSE_NONE; + } + diff --git a/src/lib/Libnet/port_forwarding.c b/src/lib/Libnet/port_forwarding.c index 7569df5084..019618ec37 100644 --- a/src/lib/Libnet/port_forwarding.c +++ b/src/lib/Libnet/port_forwarding.c @@ -18,6 +18,7 @@ #include #include #include "lib_ifl.h" +#include "lib_net.h" #include "port_forwarding.h" @@ -335,6 +336,9 @@ int x11_connect_display( char buf[1024], *cp; struct addrinfo hints, *ai, *aitop; +#ifdef BIND_OUTBOUND_SOCKETS + struct sockaddr_in local; +#endif char strport[NI_MAXSERV]; @@ -413,6 +417,14 @@ int x11_connect_display( return -1; } +#ifdef BIND_OUTBOUND_SOCKETS + if (get_local_address(local) != PBSE_NONE) + { + fprintf(stderr, "could not determine local IP address: %s", strerror(errno)); + return(-1); + } +#endif + for (ai = aitop; ai; ai = ai->ai_next) { /* Create a socket. */ @@ -424,6 +436,19 @@ int x11_connect_display( continue; } +#ifdef BIND_OUTBOUND_SOCKETS + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + + if (bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in))) + { + fprintf(stderr, "could not bind local socket: %s", strerror(errno)); + close(sock); + continue; + } + +#endif + /* Connect it to the display. */ if (connect(sock, ai->ai_addr, ai->ai_addrlen) < 0) { diff --git a/src/lib/Libnet/rm.c b/src/lib/Libnet/rm.c index b34e78682a..79551b9761 100644 --- a/src/lib/Libnet/rm.c +++ b/src/lib/Libnet/rm.c @@ -207,6 +207,7 @@ int openrm( struct sockaddr_in addr; struct addrinfo *addr_info; + struct sockaddr_in local; if (pbs_getaddrinfo(host, NULL, &addr_info) != 0) { @@ -216,18 +217,36 @@ int openrm( return(ENOENT * -1); } - memset(&addr, '\0', sizeof(addr)); +#ifdef BIND_OUTBOUND_SOCKETS + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + if (get_local_address(local) != PBSE_NONE) + { + DBPRT(("could not determine local IP address: %s", strerror(errno))); + close(stream); + return(-1); + } - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(INADDR_ANY); +#else + + memset(&local, 0, sizeof(struct sockaddr_in)); + + local.sin_family = AF_INET; + +#endif while (retries++ < MAX_RETRIES) { - rc = bindresvport(stream, &addr); + /* Attempt to bind to any reserved port that is free. If BIND_OUTBOUND_SOCKETS + * is defined, we need to make sure the socket is bound to a specific + * local IP address - passed in using the `local` structure. + */ + rc = bindresvport(stream, &local); if (rc != 0) { if (retries >= MAX_RETRIES) { + DBPRT(("could not bind local socket: %s", strerror(errno))); close(stream); return(-1*errno); } diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index a286594831..988a3b3c8b 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -100,6 +100,9 @@ extern "C" #include #include #include +#include +#include +#include #if IBM_SP2==2 /* IBM SP with PSSP 3.1 */ #include #endif /* IBM SP */ @@ -804,6 +807,10 @@ int open_demux( torque_socklen_t slen; unsigned short local_port; +#ifdef BIND_OUTBOUND_SOCKETS + struct sockaddr_in local; +#endif + memset(&remote, 0, sizeof(remote)); remote.sin_addr.s_addr = addr; remote.sin_port = htons((unsigned short)port); @@ -820,6 +827,30 @@ int open_demux( return(-1); } +#ifdef BIND_OUTBOUND_SOCKETS + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + + if (get_local_address(local) != PBSE_NONE) + { + sprintf(log_buffer, "could not determine local IP address: %s", strerror(errno)); + log_err(errno, __func__, log_buffer); + + close(sock); + return(-1); + } + + if (bind(sock, (struct sockaddr *)&local, sizeof(local))) + { + sprintf(log_buffer, "could not bind local socket: %s", strerror(errno)); + log_err(errno, __func__, log_buffer); + + close(sock); + return(-1); + } + +#endif + for (i = 0;i < RETRY;i++) { if (connect(sock, (struct sockaddr *)&remote, sizeof(remote)) == 0) diff --git a/src/test/net_common/test_uut.c b/src/test/net_common/test_uut.c index e5209d7087..0cb7686269 100644 --- a/src/test/net_common/test_uut.c +++ b/src/test/net_common/test_uut.c @@ -5,6 +5,9 @@ #include #include #include +#include +#include +#include #include "pbs_error.h" #include "net_cache.h" @@ -144,6 +147,25 @@ START_TEST(test_socket_wait_for_write) } END_TEST +START_TEST(test_get_local_address) + { + int rc; + struct sockaddr_in new_sockaddr; + char hostname[NI_MAXHOST]; + char hbuf[NI_MAXHOST]; + + gethostname(hostname, sizeof(hostname)); + + rc = get_local_address(new_sockaddr); + fail_unless(rc == PBSE_NONE); + + rc = getnameinfo((struct sockaddr*)&new_sockaddr, sizeof(new_sockaddr), hbuf, sizeof(hbuf), NULL, 0, 0); + + fail_unless(rc == 0); + fail_unless(strncmp(hostname, hbuf, NI_MAXHOST) == 0); + } +END_TEST + Suite *net_common_suite(void) { Suite *s = suite_create("net_common_suite methods"); @@ -171,6 +193,10 @@ Suite *net_common_suite(void) tcase_add_test(tc_core, test_socket_wait_for_write); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_get_local_address"); + tcase_add_test(tc_core, test_get_local_address); + suite_add_tcase(s, tc_core); + return s; } From 972ba5824472b5c0cef3a4edbbf5f9519e601f71 Mon Sep 17 00:00:00 2001 From: Daniel Saunders Date: Thu, 1 Jun 2017 14:02:27 -0600 Subject: [PATCH 286/312] TRQ-4062. Add IP address field to momctl -d output --- src/resmom/mom_main.c | 12 +++++++++++- src/test/mom_main/scaffolding.c | 5 +++++ src/test/mom_main/test_uut.c | 27 +++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index 9f07badcbe..d7027ec4ad 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -192,6 +192,7 @@ char *path_undeliv; char *path_aux; char *path_home = (char *)PBS_SERVER_HOME; char *mom_home; +char mom_ipaddr[INET_ADDRSTRLEN]; bool use_path_home = false; @@ -1645,7 +1646,8 @@ void add_diag_header( { output << "\nHost: " << mom_short_name << "/" << mom_host << " Version: "; - output << PACKAGE_VERSION << " PID: " << getpid() << "\n"; + output << PACKAGE_VERSION << " IP address: " << mom_ipaddr; + output << " PID: " << getpid() << "\n"; } /* END add_diag_header() */ @@ -4897,6 +4899,8 @@ int setup_program_environment(void) char *ptr; /* local tmp variable */ int network_retries = 0; + struct sockaddr_in network_addr; + /* must be started with real and effective uid of 0 */ if (IamRoot() == 0) { @@ -5075,6 +5079,12 @@ int setup_program_environment(void) hostc = 1; } + // Get external IP address of local node + if (get_local_address(network_addr) == 0) + { + inet_ntop(AF_INET, &(network_addr.sin_addr), mom_ipaddr, INET_ADDRSTRLEN); + } + if (!multi_mom) { log_init(NULL, mom_host); diff --git a/src/test/mom_main/scaffolding.c b/src/test/mom_main/scaffolding.c index 2da9eece4a..0f784c92bc 100644 --- a/src/test/mom_main/scaffolding.c +++ b/src/test/mom_main/scaffolding.c @@ -1146,3 +1146,8 @@ int read_all_devices() bool get_cray_taskstats; void read_rur_stats_file(const char *basepath) {} + +int get_local_address(struct sockaddr_in &new_sockaddr) + { + return PBSE_NONE; + } diff --git a/src/test/mom_main/test_uut.c b/src/test/mom_main/test_uut.c index f751b222f9..8ce2651af4 100644 --- a/src/test/mom_main/test_uut.c +++ b/src/test/mom_main/test_uut.c @@ -34,6 +34,7 @@ int process_layout_request(tcp_chan *chan); bool should_resend_obit(job *pjob, int diff); void check_job_in_mom_wait(job *pjob); void evaluate_job_in_prerun(job *pjob); +void add_diag_header(std::stringstream &output); extern attribute_def job_attr_def[]; extern int exiting_tasks; @@ -47,6 +48,10 @@ extern int flush_ret; extern int job_bailed; extern bool am_i_ms; +extern char mom_host[PBS_MAXHOSTNAME + 1]; +extern char mom_short_name[PBS_MAXHOSTNAME + 1]; +extern char mom_ipaddr[INET_ADDRSTRLEN]; + bool are_we_forking() { @@ -450,6 +455,24 @@ START_TEST(test_parse_command_line3) END_TEST +START_TEST(test_add_diag_header) + { + std::stringstream output; + std::string output_str; + + mom_ipaddr[0] = '\0'; + mom_short_name[0] = '\0'; + mom_host[0] = '\0'; + + add_diag_header(output); + + output_str = output.str(); + + fail_unless(output_str.find("IP address") != std::string::npos); + } +END_TEST + + Suite *mom_main_suite(void) { Suite *s = suite_create("mom_main_suite methods"); @@ -486,6 +509,10 @@ Suite *mom_main_suite(void) tcase_add_test(tc_core, test_parse_command_line3); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_add_diag_header"); + tcase_add_test(tc_core, test_add_diag_header); + suite_add_tcase(s, tc_core); + return s; } From 9d148ad20ad212945a8eab137e73183620ed306c Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 19 Sep 2017 15:43:18 -0600 Subject: [PATCH 287/312] Revert checkin for TRQ-3746 --- src/resmom/mom_main.c | 24 ++++++++++++------------ src/server/pbsd_init.c | 13 +++++++++++++ src/server/pbsd_main.c | 17 ----------------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index d7027ec4ad..2d8453a15f 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -4912,6 +4912,18 @@ int setup_program_environment(void) umask(022); + if (getenv("PBSLOGLEVEL") != NULL) + { + LOGLEVEL = (int)strtol(getenv("PBSLOGLEVEL"), NULL, 0); + } + + if (getenv("PBSDEBUG") != NULL) + { + DEBUGMODE = 1; + + daemonize_mom = false; + } + memset(TMOMStartInfo, 0, sizeof(pjobexec_t)*TMAX_JE); /* modify program environment */ @@ -4929,18 +4941,6 @@ int setup_program_environment(void) exit(1); } - if (getenv("PBSLOGLEVEL") != NULL) - { - LOGLEVEL = (int)strtol(getenv("PBSLOGLEVEL"), NULL, 0); - } - - if (getenv("PBSDEBUG") != NULL) - { - DEBUGMODE = 1; - - daemonize_mom = false; - } - c = getgid(); /* secure suppl. groups */ diff --git a/src/server/pbsd_init.c b/src/server/pbsd_init.c index 50dba27043..ec595741c9 100644 --- a/src/server/pbsd_init.c +++ b/src/server/pbsd_init.c @@ -2270,6 +2270,19 @@ int pbsd_init( memset(&hints, 0, sizeof(hints)); hints.ai_flags = AI_CANONNAME; + /* The following is code to reduce security risks */ + if (use_path_home == true) + { + path_pbs_environment = build_path(path_home, "pbs_environment", ""); + if (setup_env(path_pbs_environment) == -1) + { + return(-1); + } + } + else if (setup_env(PBS_ENVIRON) == -1) + { + return(-1); + } gid = getgid(); diff --git a/src/server/pbsd_main.c b/src/server/pbsd_main.c index 7d719c723c..c6db8b05b0 100644 --- a/src/server/pbsd_main.c +++ b/src/server/pbsd_main.c @@ -111,7 +111,6 @@ #include "../lib/Liblog/pbs_log.h" #include "../lib/Liblog/log_event.h" #include "../lib/Liblog/chk_file_sec.h" -#include "../lib/Liblog/setup_env.h" #include "lib_ifl.h" #include "server_limits.h" #include "attribute.h" @@ -172,7 +171,6 @@ extern int RPPConfigure(int, int); extern void acct_cleanup(long); void stream_eof(int, u_long, uint16_t, int); extern void scheduler_close(); -extern char *build_path(char *, const char *, const char *); #ifndef MAX_PATH_LEN #define MAX_PATH_LEN 256 @@ -1621,21 +1619,6 @@ int main( memset(&server, 0, sizeof(struct server)); log_init(NULL, NULL); initialize_globals(); - - /* The following is code to reduce security risks */ - if (use_path_home == true) - { - path_pbs_environment = build_path(path_home, "pbs_environment", ""); - if (setup_env(path_pbs_environment) == -1) - { - exit(1); - } - } - else if (setup_env(PBS_ENVIRON) == -1) - { - exit(1); - } - set_globals_from_environment(); /* set standard umask */ From 2013fc8b97ebf3bbaccad089d14ec203a89f32d0 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 21 Sep 2017 12:10:20 -0600 Subject: [PATCH 288/312] TRQ-4072 - do not bind socket to local mom --- src/lib/Libifl/tm.c | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/lib/Libifl/tm.c b/src/lib/Libifl/tm.c index d12a8d79ce..a08cf384bb 100644 --- a/src/lib/Libifl/tm.c +++ b/src/lib/Libifl/tm.c @@ -484,11 +484,6 @@ static int localmom(void) struct linger ltime; -#ifdef BIND_OUTBOUND_SOCKETS - struct sockaddr_in local; -#endif - - if (local_conn >= 0) { return(local_conn); /* already have open connection */ @@ -533,27 +528,6 @@ static int localmom(void) #endif -#ifdef BIND_OUTBOUND_SOCKETS - - /* Bind to the IP address associated with the hostname, in case there are - * muliple possible source IPs for this destination.*/ - - if (get_local_address(local) != 0) - { - TM_DBPRT(("tm_init: unable to determine local IP address")); - close(sock); - return(-1); - } - - if (bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in)) != 0) - { - TM_DBPRT(("tm_init: unable to bind to local IP address")); - close(sock); - return(-1); - } -#endif - - /* make sure data goes out */ ltime.l_onoff = 1; From ad151fe247602da912cebf8e13b6d684d26a674e Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 22 Sep 2017 15:23:03 -0600 Subject: [PATCH 289/312] TRQ-4072 - conditionally bind outbound sockets --- src/lib/Libifl/trq_auth.c | 16 ++++++++++------ src/lib/Libnet/lib_net.h | 1 + src/lib/Libnet/net_common.c | 12 ++++++++++++ src/lib/Libnet/port_forwarding.c | 12 ++++++++---- src/resmom/start_exec.c | 14 +++++++++----- src/test/start_exec/scaffolding.c | 16 ++++++++++++++++ src/test/trq_auth/scaffolding.c | 17 +++++++++++++++++ 7 files changed, 73 insertions(+), 15 deletions(-) diff --git a/src/lib/Libifl/trq_auth.c b/src/lib/Libifl/trq_auth.c index 524b14c404..05be9780b1 100644 --- a/src/lib/Libifl/trq_auth.c +++ b/src/lib/Libifl/trq_auth.c @@ -380,13 +380,17 @@ int trq_simple_connect( #ifdef BIND_OUTBOUND_SOCKETS - rc = bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in)); - if (rc != 0) + // only bind if not localhost + if (!islocalhostaddr(&local)) { - fprintf(stderr, "could not bind local socket: %s", strerror(errno)); - rc = PBSE_SYSTEM; - close(sock); - continue; + rc = bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in)); + if (rc != 0) + { + fprintf(stderr, "could not bind local socket: %s", strerror(errno)); + rc = PBSE_SYSTEM; + close(sock); + continue; + } } #endif diff --git a/src/lib/Libnet/lib_net.h b/src/lib/Libnet/lib_net.h index 4e008ecfb8..dd732a247d 100644 --- a/src/lib/Libnet/lib_net.h +++ b/src/lib/Libnet/lib_net.h @@ -53,6 +53,7 @@ int socket_close(int socket); int pbs_getaddrinfo(const char *pNode,struct addrinfo *pHints,struct addrinfo **ppAddrInfoOut); int connect_to_trqauthd(int *sock); int get_local_address(struct sockaddr_in &new_sockaddr); +bool islocalhostaddr(struct sockaddr_in*); /* from file server_core.c */ diff --git a/src/lib/Libnet/net_common.c b/src/lib/Libnet/net_common.c index dc4e5702d4..7b36d31aa1 100644 --- a/src/lib/Libnet/net_common.c +++ b/src/lib/Libnet/net_common.c @@ -1039,6 +1039,18 @@ int connect_to_trqauthd( return(rc); } +/* + * islocalhostaddr() - check if sockaddr_in holds the locahost address + * + */ +bool islocalhostaddr(struct sockaddr_in *saip) + { + if (saip == NULL) + return(false); + + // is 127.0.0.1? + return((saip->sin_addr.s_addr == (uint32_t)((127 << 24) + 1))); + } /* * get_local_address() - lookup the local IP address associated with the local, diff --git a/src/lib/Libnet/port_forwarding.c b/src/lib/Libnet/port_forwarding.c index 019618ec37..0e682a1d2b 100644 --- a/src/lib/Libnet/port_forwarding.c +++ b/src/lib/Libnet/port_forwarding.c @@ -440,11 +440,15 @@ int x11_connect_display( /* Bind to the IP address associated with the hostname, in case there are * muliple possible source IPs for this destination.*/ - if (bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in))) + // don't bind localhost addr + if (!islocalhostaddr(&local)) { - fprintf(stderr, "could not bind local socket: %s", strerror(errno)); - close(sock); - continue; + if (bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in))) + { + fprintf(stderr, "could not bind local socket: %s", strerror(errno)); + close(sock); + continue; + } } #endif diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index bcfb482603..a376a49059 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -840,13 +840,17 @@ int open_demux( return(-1); } - if (bind(sock, (struct sockaddr *)&local, sizeof(local))) + // do not bind if lochost addr + if (!islocalhostaddr(&local)) { - sprintf(log_buffer, "could not bind local socket: %s", strerror(errno)); - log_err(errno, __func__, log_buffer); + if (bind(sock, (struct sockaddr *)&local, sizeof(local))) + { + sprintf(log_buffer, "could not bind local socket: %s", strerror(errno)); + log_err(errno, __func__, log_buffer); - close(sock); - return(-1); + close(sock); + return(-1); + } } #endif diff --git a/src/test/start_exec/scaffolding.c b/src/test/start_exec/scaffolding.c index b5a2222e1b..be08b26105 100644 --- a/src/test/start_exec/scaffolding.c +++ b/src/test/start_exec/scaffolding.c @@ -1116,3 +1116,19 @@ int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const return(0); } + +int get_local_address(struct sockaddr_in &new_sockaddr) + { + // return 127.0.0.1 + new_sockaddr.sin_addr.s_addr = (uint32_t)2130706433; + return(0); + } + +bool islocalhostaddr(struct sockaddr_in *saip) + { + if (saip == NULL) + return(false); + + // is 127.0.0.1? + return((saip->sin_addr.s_addr == (uint32_t)((127 << 24) + 1))); +} diff --git a/src/test/trq_auth/scaffolding.c b/src/test/trq_auth/scaffolding.c index ca59e8e5e9..df8fb69a58 100644 --- a/src/test/trq_auth/scaffolding.c +++ b/src/test/trq_auth/scaffolding.c @@ -657,3 +657,20 @@ struct passwd *get_password_entry_by_uid(char **user_buf, uid_t uid) void free_pwnam(struct passwd *pwdp, char *buf) { } + +int get_local_address(struct sockaddr_in &new_sockaddr) + { + // return 127.0.0.1 + new_sockaddr.sin_addr.s_addr = (uint32_t) ((127 << 24) + 1); + return(0); + } + +bool islocalhostaddr(struct sockaddr_in *saip) + { + if (saip == NULL) + return(false); + + // is 127.0.0.1? + return((saip->sin_addr.s_addr == (uint32_t)((127 << 24) + 1))); +} + From d767357bcca65ef228c40d3e6d760af7c2b6f821 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 26 Sep 2017 09:50:00 -0600 Subject: [PATCH 290/312] Update unit test. --- src/test/req_signal/test_uut.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/req_signal/test_uut.c b/src/test/req_signal/test_uut.c index 5e25b4926b..d201ea4895 100644 --- a/src/test/req_signal/test_uut.c +++ b/src/test/req_signal/test_uut.c @@ -47,6 +47,8 @@ void init_attr_def() START_TEST(test_issue_signal) { job pjob; + + memset(&pjob, 0, sizeof(job)); strcpy(pjob.ji_qs.ji_jobid, "1.nalthis"); job *ptr = &pjob; From 78efae7e8a5a4cd0984b21a8d4ad364589ee981d Mon Sep 17 00:00:00 2001 From: knielson Date: Thu, 5 May 2016 15:54:58 -0600 Subject: [PATCH 291/312] Removed the word Error from an information message on the MOM. --- src/resmom/mom_comm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index 7b34686076..f75733f841 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -6391,7 +6391,7 @@ void im_request( chan->sock = -1; snprintf(log_buffer, LOCAL_LOG_BUF_SIZE, - "Error response received from client %s (%d) jobid %s", + "response received from client %s (%d) jobid %s", netaddr(pSockAddr), sender_port, jobid); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, jobid, log_buffer); From 9a94da389aac8c8d224c02560730c041ed882c32 Mon Sep 17 00:00:00 2001 From: David Beer Date: Tue, 12 Jan 2016 15:50:54 -0700 Subject: [PATCH 292/312] Fix an error when reserving mics for jobs. Conflicts: src/server/node_manager.c --- src/server/node_manager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index dc7dfcbf8a..35a5860aa8 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -2538,7 +2538,7 @@ int save_node_for_adding( /* count off the number we have reserved */ pnode->nd_np_to_be_used += req.ppn; pnode->nd_ngpus_to_be_used += req.gpu; - pnode->nd_nmics_to_be_used -= req.mic; + pnode->nd_nmics_to_be_used += req.mic; return(PBSE_NONE); } /* END save_node_for_adding */ From 177cf849a6af94275bd2dc2f3797ca2993b7c3f5 Mon Sep 17 00:00:00 2001 From: Auto Git Date: Tue, 26 Sep 2017 11:57:01 -0600 Subject: [PATCH 293/312] Update version to 6.1.2 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index b2a31722bd..195d6bbe23 100644 --- a/configure.ac +++ b/configure.ac @@ -33,7 +33,7 @@ dnl added library functionality to testing dnl by John Rosenquist (jrosenquist@adaptivecomputing.com) AC_PREREQ(2.53) -AC_INIT([torque], [6.1], [torqueusers@supercluster.org]) +AC_INIT([torque], [6.1.2], [torqueusers@supercluster.org]) AC_REVISION($Revision$) AC_CONFIG_SRCDIR([src/cmds/qrun.c]) AC_CONFIG_AUX_DIR([buildutils]) From 5dc37e51fe8ecf3dd2f49766f5b41dcaa5798c4b Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 29 Sep 2017 15:12:52 -0600 Subject: [PATCH 294/312] TRQ-4070 - use vectors in port fowarding code --- src/cmds/qsub_functions.c | 14 ++-- src/include/port_forwarding.h | 4 +- src/lib/Libnet/port_forwarding.c | 126 +++++++++++++++---------------- src/resmom/mom_inter.c | 45 +++++------ 4 files changed, 87 insertions(+), 102 deletions(-) diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index 33776770b2..eabbf66bde 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -39,6 +39,8 @@ #include #include +#include + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif @@ -2278,20 +2280,18 @@ void x11handler( int param_sock) { - struct pfwdsock *socks; + std::vector *socks = new std::vector(NUM_SOCKS); int n; char *display; - calloc_or_fail((char **)&socks, sizeof(struct pfwdsock) * NUM_SOCKS, "x11handler"); - for (n = 0;n < NUM_SOCKS;n++) - (socks + n)->active = 0; + socks->at(n).active = 0; - (socks + 0)->sock = param_sock; + socks->at(0).sock = param_sock; - (socks + 0)->active = 1; + socks->at(0).active = 1; - (socks + 0)->listening = 1; + socks->at(0).listening = 1; /* Try to open a socket for the local X server. */ display = getenv("DISPLAY"); diff --git a/src/include/port_forwarding.h b/src/include/port_forwarding.h index ef88000b79..04bbff0847 100644 --- a/src/include/port_forwarding.h +++ b/src/include/port_forwarding.h @@ -6,6 +6,8 @@ #define MAX_DISPLAYS 500 #define X11OFFSET 50 +#include + /* derived from XF4/xc/lib/dps/Xlibnet.h */ #ifndef X_UNIX_PATH @@ -33,7 +35,7 @@ struct pfwdsock char buff[BUF_SIZE]; }; -void port_forwarder(struct pfwdsock *, int(*connfunc)(char *phost, long pport, char *), char*, int, char *); +void port_forwarder(std::vector *, int(*connfunc)(char *phost, long pport, char *), char*, int, char *); void set_nodelay(int); int connect_local_xsocket(u_int); int x11_connect_display(char *, long, char *); diff --git a/src/lib/Libnet/port_forwarding.c b/src/lib/Libnet/port_forwarding.c index 0e682a1d2b..e03351dea4 100644 --- a/src/lib/Libnet/port_forwarding.c +++ b/src/lib/Libnet/port_forwarding.c @@ -19,6 +19,7 @@ #include #include "lib_ifl.h" #include "lib_net.h" +#include #include "port_forwarding.h" @@ -34,7 +35,7 @@ void port_forwarder( - struct pfwdsock *socks, + std::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, @@ -48,51 +49,42 @@ void port_forwarder( int rc; struct sockaddr_in from; torque_socklen_t fromlen; - struct pollfd *PollArray; - - - - PollArray = (struct pollfd *)malloc(NUM_SOCKS * sizeof(struct pollfd)); - - if (PollArray == NULL) - { - return; // no memory - } + std::vector PollArray(NUM_SOCKS); while (1) { for (n = 0; n < NUM_SOCKS; n++) { // clear the entry - PollArray[n].fd = -1; - PollArray[n].events = 0; - PollArray[n].revents = 0; + PollArray.at(n).fd = -1; + PollArray.at(n).events = 0; + PollArray.at(n).revents = 0; - if (!(socks + n)->active) + if (!socks->at(n).active) continue; - if ((socks + n)->listening) + if (socks->at(n).listening) { - PollArray[n].fd = (socks + n)->sock; - PollArray[n].events = POLLIN; + PollArray.at(n).fd = socks->at(n).sock; + PollArray.at(n).events = POLLIN; } else { - if ((socks + n)->bufavail < BUF_SIZE) + if (socks->at(n).bufavail < BUF_SIZE) { - PollArray[n].fd = (socks + n)->sock; - PollArray[n].events = POLLIN; + PollArray.at(n).fd = socks->at(n).sock; + PollArray.at(n).events = POLLIN; } - if ((socks + ((socks + n)->peer))->bufavail - (socks + ((socks + n)->peer))->bufwritten > 0) + if (socks->at(socks->at(n).peer).bufavail - socks->at(socks->at(n).peer).bufwritten > 0) { - PollArray[n].fd = (socks + n)->sock; - PollArray[n].events |= POLLOUT; + PollArray.at(n).fd = socks->at(n).sock; + PollArray.at(n).events |= POLLOUT; } } } - num_events = poll(PollArray, NUM_SOCKS, -1); + num_events = poll(&PollArray[0], NUM_SOCKS, -1); if ((num_events == -1) && (errno == EINTR)) continue; @@ -107,30 +99,30 @@ void port_forwarder( for (n = 0; (num_events > 0) && (n < NUM_SOCKS); n++) { // skip entry with no return events - if (PollArray[n].revents == 0) + if (PollArray.at(n).revents == 0) continue; // decrement the count of events returned num_events--; - if (!(socks + n)->active) + if (!socks->at(n).active) continue; - if ((PollArray[n].revents & POLLIN)) + if ((PollArray.at(n).revents & POLLIN)) { - if ((socks + n)->listening) + if (socks->at(n).listening) { int newsock = 0, peersock = 0; fromlen = sizeof(from); - if ((sock = accept((socks + n)->sock, (struct sockaddr *) & from, &fromlen)) < 0) + if ((sock = accept(socks->at(n).sock, (struct sockaddr *) & from, &fromlen)) < 0) { if ((errno == EAGAIN) || (errno == EWOULDBLOCK) || (errno == EINTR) || (errno == ECONNABORTED)) continue; - close((socks + n)->sock); + close(socks->at(n).sock); - (socks + n)->active = 0; + socks->at(n).active = 0; continue; } @@ -139,7 +131,7 @@ void port_forwarder( for (n2 = 0; n2 < NUM_SOCKS; n2++) { - if ((socks + n2)->active || (((socks + n2)->peer != 0) && (socks + ((socks + n2)->peer))->active)) + if (socks->at(n2).active || ((socks->at(n2).peer != 0) && socks->at(socks->at(n2).peer).active)) continue; if (newsock == 0) @@ -150,56 +142,56 @@ void port_forwarder( break; } - (socks + newsock)->sock = (socks + peersock)->remotesock = sock; - (socks + newsock)->listening = (socks + peersock)->listening = 0; - (socks + newsock)->active = (socks + peersock)->active = 1; - (socks + newsock)->peer = (socks + peersock)->sock = connfunc(phost, pport, EMsg); - (socks + newsock)->bufwritten = (socks + peersock)->bufwritten = 0; - (socks + newsock)->bufavail = (socks + peersock)->bufavail = 0; - (socks + newsock)->buff[0] = (socks + peersock)->buff[0] = '\0'; - (socks + newsock)->peer = peersock; - (socks + peersock)->peer = newsock; + socks->at(newsock).sock = socks->at(peersock).remotesock = sock; + socks->at(newsock).listening = socks->at(peersock).listening = 0; + socks->at(newsock).active = socks->at(peersock).active = 1; + socks->at(newsock).peer = socks->at(peersock).sock = connfunc(phost, pport, EMsg); + socks->at(newsock).bufwritten = socks->at(peersock).bufwritten = 0; + socks->at(newsock).bufavail = socks->at(peersock).bufavail = 0; + socks->at(newsock).buff[0] = socks->at(peersock).buff[0] = '\0'; + socks->at(newsock).peer = peersock; + socks->at(peersock).peer = newsock; } else { /* non-listening socket to be read */ rc = read_ac_socket( - (socks + n)->sock, - (socks + n)->buff + (socks + n)->bufavail, - BUF_SIZE - (socks + n)->bufavail); + socks->at(n).sock, + socks->at(n).buff + socks->at(n).bufavail, + BUF_SIZE - socks->at(n).bufavail); if (rc < 1) { - shutdown((socks + n)->sock, SHUT_RDWR); - close((socks + n)->sock); - (socks + n)->active = 0; + shutdown(socks->at(n).sock, SHUT_RDWR); + close(socks->at(n).sock); + socks->at(n).active = 0; } else { - (socks + n)->bufavail += rc; + socks->at(n).bufavail += rc; } } } /* END if read */ - if ((PollArray[n].revents & POLLOUT)) + if ((PollArray.at(n).revents & POLLOUT)) { - int peer = (socks + n)->peer; + int peer = socks->at(n).peer; rc = write_ac_socket( - (socks + n)->sock, - (socks + peer)->buff + (socks + peer)->bufwritten, - (socks + peer)->bufavail - (socks + peer)->bufwritten); + socks->at(n).sock, + socks->at(peer).buff + socks->at(peer).bufwritten, + socks->at(peer).bufavail - socks->at(peer).bufwritten); if (rc < 1) { - shutdown((socks + n)->sock, SHUT_RDWR); - close((socks + n)->sock); - (socks + n)->active = 0; + shutdown(socks->at(n).sock, SHUT_RDWR); + close(socks->at(n).sock); + socks->at(n).active = 0; } else { - (socks + peer)->bufwritten += rc; + socks->at(peer).bufwritten += rc; } } /* END if write */ } /* END foreach fd */ @@ -210,22 +202,22 @@ void port_forwarder( { int peer; - if (!(socks + n)->active || (socks + n)->listening) + if (!socks->at(n).active || socks->at(n).listening) continue; - peer = (socks + n)->peer; + peer = socks->at(n).peer; - if ((socks + n)->bufwritten == (socks + n)->bufavail) + if (socks->at(n).bufwritten == socks->at(n).bufavail) { - (socks + n)->bufwritten = (socks + n)->bufavail = 0; + socks->at(n).bufwritten = socks->at(n).bufavail = 0; } - if (!(socks + peer)->active && ((socks + peer)->bufwritten == (socks + peer)->bufavail)) + if (!socks->at(peer).active && (socks->at(peer).bufwritten == socks->at(peer).bufavail)) { - shutdown((socks + n)->sock, SHUT_RDWR); - close((socks + n)->sock); + shutdown(socks->at(n).sock, SHUT_RDWR); + close(socks->at(n).sock); - (socks + n)->active = 0; + socks->at(n).active = 0; } } } @@ -233,7 +225,7 @@ void port_forwarder( // never executed but should satisfy code profilers looking // for a malloc/free pairing - free(PollArray); + delete(socks); } /* END port_forwarder() */ diff --git a/src/resmom/mom_inter.c b/src/resmom/mom_inter.c index 898ab6f70e..c8b1d0cf94 100644 --- a/src/resmom/mom_inter.c +++ b/src/resmom/mom_inter.c @@ -538,31 +538,22 @@ int x11_create_display( FILE *f; pid_t childpid; - struct pfwdsock *socks; + std::vector *socks = new std::vector(NUM_SOCKS); const char *homeenv = "HOME"; *display = '\0'; - if ((socks = (struct pfwdsock *)calloc(sizeof(struct pfwdsock), NUM_SOCKS)) == NULL) - { - /* FAILURE - cannot alloc memory */ - - fprintf(stderr,"ERROR: could not calloc!\n"); - - return(-1); - } - if (put_env_var(homeenv, homedir)) { /* FAILURE */ - free(socks); + delete(socks); fprintf(stderr, "ERROR: could not insert %s into environment\n", homeenv); return(-1); } for (n = 0;n < NUM_SOCKS;n++) - socks[n].active = 0; + socks->at(n).active = 0; x11proto[0] = x11data[0] = '\0'; @@ -578,7 +569,7 @@ int x11_create_display( n, strerror(errno)); - free(socks); + delete(socks); return(-1); } @@ -601,7 +592,7 @@ int x11_create_display( fprintf(stderr, "getaddrinfo: %.100s\n", gai_strerror(gaierr)); - free(socks); + delete(socks); return(-1); } @@ -626,7 +617,7 @@ int x11_create_display( fprintf(stderr, "socket: %.100s\n", strerror(errno)); - free(socks); + delete(socks); freeaddrinfo(aitop); return (-1); @@ -666,7 +657,7 @@ int x11_create_display( for (n = 0; n < num_socks; n++) { - close(socks[n].sock); + close(socks->at(n).sock); } num_socks = 0; @@ -674,8 +665,8 @@ int x11_create_display( break; } - socks[num_socks].sock = sock; - socks[num_socks].active = 1; + socks->at(num_socks).sock = sock; + socks->at(num_socks).active = 1; num_socks++; #ifndef DONT_TRY_OTHER_AF @@ -706,7 +697,7 @@ int x11_create_display( { fprintf(stderr, "Failed to allocate internet-domain X11 display socket.\n"); - free(socks); + delete(socks); return(-1); } @@ -716,21 +707,21 @@ int x11_create_display( for (n = 0;n < num_socks;n++) { DBPRT(("listening on fd %d\n", - socks[n].sock)); + socks->at(n).sock)); - if (listen(socks[n].sock,TORQUE_LISTENQUEUE) < 0) + if (listen(socks->at(n).sock,TORQUE_LISTENQUEUE) < 0) { fprintf(stderr,"listen: %.100s\n", strerror(errno)); - close(socks[n].sock); + close(socks->at(n).sock); - free(socks); + delete(socks); return(-1); } - socks[n].listening = 1; + socks->at(n).listening = 1; } /* END for (n) */ /* setup local xauth */ @@ -766,14 +757,14 @@ int x11_create_display( fprintf(stderr, "could not run %s\n", cmd); - free(socks); + delete(socks); return(-1); } if ((childpid = fork()) > 0) { - free(socks); + delete(socks); DBPRT(("successful x11 init, returning display %d\n", display_number)); @@ -785,7 +776,7 @@ int x11_create_display( { fprintf(stderr, "failed x11 init fork\n"); - free(socks); + delete(socks); return(-1); } From ccab25c2a10e435ded8aa1351fbe8fe68446eef2 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 29 Sep 2017 16:47:38 -0600 Subject: [PATCH 295/312] TRQ-4070 - fix unit tests --- src/lib/Libnet/lib_net.h | 2 +- src/test/mom_inter/scaffolding.c | 2 +- src/test/qsub_functions/scaffolding.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/Libnet/lib_net.h b/src/lib/Libnet/lib_net.h index dd732a247d..beca183ab0 100644 --- a/src/lib/Libnet/lib_net.h +++ b/src/lib/Libnet/lib_net.h @@ -92,7 +92,7 @@ char *netaddr_pbs_net_t(pbs_net_t ipadd); void net_add_close_func(int, void (*func)(int)); /* from file port_forwarding.c */ -void port_forwarder(struct pfwdsock *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg); +void port_forwarder(std::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg); void set_nodelay(int fd); int connect_local_xsocket(u_int dnr); int x11_connect_display(char *display, long alsounused, char *EMsg); diff --git a/src/test/mom_inter/scaffolding.c b/src/test/mom_inter/scaffolding.c index 9a3a295b0e..3d47c3126c 100644 --- a/src/test/mom_inter/scaffolding.c +++ b/src/test/mom_inter/scaffolding.c @@ -18,7 +18,7 @@ ssize_t read_blocking_socket(int fd, void *buf, ssize_t count) return(0); } -void port_forwarder( struct pfwdsock *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) +void port_forwarder(td::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) { fprintf(stderr, "The call to port_forwarder needs to be mocked!!\n"); exit(1); diff --git a/src/test/qsub_functions/scaffolding.c b/src/test/qsub_functions/scaffolding.c index d50329969a..ff10fccab3 100644 --- a/src/test/qsub_functions/scaffolding.c +++ b/src/test/qsub_functions/scaffolding.c @@ -123,7 +123,7 @@ int locate_job(char *job_id, char *parent_server, char *located_server) exit(1); } -void port_forwarder( struct pfwdsock *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) +void port_forwarder(std::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) { fprintf(stderr, "The call to port_forwarder to be mocked!!\n"); exit(1); From 2592589f0c0ad4ad9a027fc39b044785d786e3b7 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 29 Sep 2017 17:18:18 -0600 Subject: [PATCH 296/312] TRQ-4070 - fix unit test typo --- src/test/mom_inter/scaffolding.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/mom_inter/scaffolding.c b/src/test/mom_inter/scaffolding.c index 3d47c3126c..a4cc3d5dde 100644 --- a/src/test/mom_inter/scaffolding.c +++ b/src/test/mom_inter/scaffolding.c @@ -18,7 +18,7 @@ ssize_t read_blocking_socket(int fd, void *buf, ssize_t count) return(0); } -void port_forwarder(td::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) +void port_forwarder(std::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) { fprintf(stderr, "The call to port_forwarder needs to be mocked!!\n"); exit(1); From 18ec517b75db0d8df00361f610461e6cfd04e834 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 9 Oct 2017 13:41:14 -0600 Subject: [PATCH 297/312] Fix Valgrind issues. --- src/resmom/mom_server.c | 1 + src/resmom/parse_config.c | 3 ++- src/server/array_func.c | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/resmom/mom_server.c b/src/resmom/mom_server.c index d151f43bc2..eb5c48270e 100644 --- a/src/resmom/mom_server.c +++ b/src/resmom/mom_server.c @@ -893,6 +893,7 @@ void gen_gen( s += "="; s += value; status.push_back(s); + free((void *)value); } } /* else if (ap) */ diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index 8c4bb8585d..0176e478b0 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -2436,7 +2436,8 @@ void reset_config_vars() { delete_link(&pva->va_link); - free(pva->va_cmd); + if (pva->va_value != NULL) free(pva->va_value); + if (pva->va_cmd != NULL) free(pva->va_cmd); free(pva); } } // END reset_config_vars() diff --git a/src/server/array_func.c b/src/server/array_func.c index b27dcf03e0..ba24c0a612 100644 --- a/src/server/array_func.c +++ b/src/server/array_func.c @@ -1187,6 +1187,7 @@ int setup_array_struct( { pa_mutex.unlock(); array_delete(pjob->ji_qs.ji_jobid); + delete pa; return(rc); } @@ -1195,6 +1196,8 @@ int setup_array_struct( { pa_mutex.unlock(); array_delete(pjob->ji_qs.ji_jobid); + delete pa; + return(rc); } @@ -1204,6 +1207,8 @@ int setup_array_struct( insert_array(pa); + delete pa; + return(PBSE_NONE); } /* END setup_array_struct() */ From 9f58a265ff393c8d02dc0e5dcd3548ecd00d0694 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 9 Oct 2017 15:29:04 -0600 Subject: [PATCH 298/312] Fix free() problem with previous valgrind commit. --- src/resmom/mom_server.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/resmom/mom_server.c b/src/resmom/mom_server.c index eb5c48270e..d151f43bc2 100644 --- a/src/resmom/mom_server.c +++ b/src/resmom/mom_server.c @@ -893,7 +893,6 @@ void gen_gen( s += "="; s += value; status.push_back(s); - free((void *)value); } } /* else if (ap) */ From 71a0eb3b10f65de01cf712229516ac239e05df2c Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 10 Oct 2017 10:25:50 -0600 Subject: [PATCH 299/312] Back out recent valgrind fixes. --- src/resmom/parse_config.c | 3 +-- src/server/array_func.c | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index 0176e478b0..8c4bb8585d 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -2436,8 +2436,7 @@ void reset_config_vars() { delete_link(&pva->va_link); - if (pva->va_value != NULL) free(pva->va_value); - if (pva->va_cmd != NULL) free(pva->va_cmd); + free(pva->va_cmd); free(pva); } } // END reset_config_vars() diff --git a/src/server/array_func.c b/src/server/array_func.c index ba24c0a612..b27dcf03e0 100644 --- a/src/server/array_func.c +++ b/src/server/array_func.c @@ -1187,7 +1187,6 @@ int setup_array_struct( { pa_mutex.unlock(); array_delete(pjob->ji_qs.ji_jobid); - delete pa; return(rc); } @@ -1196,8 +1195,6 @@ int setup_array_struct( { pa_mutex.unlock(); array_delete(pjob->ji_qs.ji_jobid); - delete pa; - return(rc); } @@ -1207,8 +1204,6 @@ int setup_array_struct( insert_array(pa); - delete pa; - return(PBSE_NONE); } /* END setup_array_struct() */ From c906ae44515d33d7e77d206260b5fbb8e4c77f61 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 12 Oct 2017 15:07:51 -0600 Subject: [PATCH 300/312] Fix previous check in for TRQ-4065 --- src/server/pbsd_init.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/server/pbsd_init.c b/src/server/pbsd_init.c index ec595741c9..0233800586 100644 --- a/src/server/pbsd_init.c +++ b/src/server/pbsd_init.c @@ -2055,9 +2055,6 @@ int cleanup_recovered_arrays() } } - std::string array_id(pa->ai_qs.parent_id); - unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); - array_delete(array_id.c_str()); if (pa != NULL) { std::string array_id(pa->ai_qs.parent_id); From 0a238e83f9f59910ddb9f586de26177933f71765 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 16 Oct 2017 09:43:36 -0600 Subject: [PATCH 301/312] TRQ-4101 - add Delegate=yes to pbs_mom unit file --- contrib/systemd/pbs_mom.service.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/systemd/pbs_mom.service.in b/contrib/systemd/pbs_mom.service.in index a13513fb8d..f832852a02 100644 --- a/contrib/systemd/pbs_mom.service.in +++ b/contrib/systemd/pbs_mom.service.in @@ -8,6 +8,8 @@ After=network.target local-fs.target syslog.target Type=simple User=root +Delegate=yes + LimitNOFILE=32768 LimitMEMLOCK=infinity LimitSTACK=12582912 From 7f5879e7ff8d7e429b2798b7fda5f5e4978c49c4 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Tue, 17 Oct 2017 15:47:02 -0600 Subject: [PATCH 302/312] TRQ-4128 - avoid deadlock when looking for orphaned reservations --- src/server/track_alps_reservations.cpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/server/track_alps_reservations.cpp b/src/server/track_alps_reservations.cpp index ffbc7ae854..159cd8a287 100644 --- a/src/server/track_alps_reservations.cpp +++ b/src/server/track_alps_reservations.cpp @@ -187,15 +187,25 @@ bool reservation_holder::is_orphaned( bool orphaned = false; job *pjob; std::map::iterator it; + int internal_jid = -1; + bool res_found = false; pthread_mutex_lock(&this->rh_mutex); it = this->reservations.find(rsv_id); if (it != this->reservations.end()) { - job_id = job_mapper.get_name(it->second.internal_job_id); + internal_jid = it->second.internal_job_id; + res_found = true; + } + + pthread_mutex_unlock(&this->rh_mutex); + + if (res_found) + { + job_id = job_mapper.get_name(internal_jid); - if ((pjob = svr_find_job_by_id(it->second.internal_job_id)) != NULL) + if ((pjob = svr_find_job_by_id(internal_jid)) != NULL) { if (pjob->ji_qs.ji_state == JOB_STATE_COMPLETE) orphaned = true; @@ -215,6 +225,8 @@ bool reservation_holder::is_orphaned( if (orphaned == true) { + pthread_mutex_lock(&this->rh_mutex); + if (this->orphaned_reservations.find(rsv_id) != this->orphaned_reservations.end()) { // Only send one release reservation request at a time. @@ -225,9 +237,9 @@ bool reservation_holder::is_orphaned( // Record this so we only send one at a time this->orphaned_reservations.insert(rsv_id); } - } - pthread_mutex_unlock(&this->rh_mutex); + pthread_mutex_unlock(&this->rh_mutex); + } return(orphaned); } /* END is_orphaned() */ From 54f21e0cf51886f40b1906d68646e562fa6806ab Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 26 Oct 2017 10:50:53 -0600 Subject: [PATCH 303/312] TRQ-3978 - use NVML API rather than hwloc for info about NVIDIA devices --- src/include/machine.hpp | 2 +- src/lib/Libutils/numa_pci_device.cpp | 47 +++++- src/resmom/accelerators_numa.cpp | 209 ++++++++------------------- src/server/pbsd_main.c | 2 +- 4 files changed, 98 insertions(+), 162 deletions(-) diff --git a/src/include/machine.hpp b/src/include/machine.hpp index 845f7d5a71..d0fdf631d8 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -60,7 +60,7 @@ class PCI_Device #endif #ifdef NVIDIA_GPUS - void initializeGpu(int idx, hwloc_topology_t topology); + void initializeGpu(int idx); #endif void displayAsString(stringstream &out) const; void setName(const string &name); diff --git a/src/lib/Libutils/numa_pci_device.cpp b/src/lib/Libutils/numa_pci_device.cpp index bf8dcdd960..f53f1e9f71 100644 --- a/src/lib/Libutils/numa_pci_device.cpp +++ b/src/lib/Libutils/numa_pci_device.cpp @@ -68,7 +68,44 @@ PCI_Device::~PCI_Device() int PCI_Device::initializePCIDevice(hwloc_obj_t device_obj, int idx, hwloc_topology_t topology) { - if (device_obj != NULL) + // see if we are initializing an NVIDIA PCI device only + if (device_obj == NULL) + { +#if defined(NVIDIA_GPUS) && defined(NVML_API) + // use NVML library to look up info about the device + + nvmlDevice_t gpu; + char *buf; + + // get the device handle so the name can be looked up + if (nvmlDeviceGetHandleByIndex(idx, &gpu) != NVML_SUCCESS) + return(-1); + + // allocate space for the name + if ((buf = (char *)malloc(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE)) == NULL) + return(-1); + + // look up the name + if (nvmlDeviceGetName(gpu, buf, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE) != NVML_SUCCESS) + { + free(buf); + return(-1); + } + + // copy the name + name = buf; + + // free the buffer + free(buf); + + // copy the id + id = idx; +#else + // NULL hwloc object passed in + return(-1); +#endif + } + else { id = device_obj->logical_index; name = device_obj->name; @@ -78,19 +115,13 @@ int PCI_Device::initializePCIDevice(hwloc_obj_t device_obj, int idx, hwloc_topol info_value = device_obj->infos->value; } } - else - { - id = idx; - name = "Unknown"; - } - #ifdef MIC this->initializeMic(idx, topology); #endif #ifdef NVIDIA_GPUS - this->initializeGpu(idx, topology); + this->initializeGpu(idx); #endif return(PBSE_NONE); } diff --git a/src/resmom/accelerators_numa.cpp b/src/resmom/accelerators_numa.cpp index e890f0aafe..28fcfa25f5 100644 --- a/src/resmom/accelerators_numa.cpp +++ b/src/resmom/accelerators_numa.cpp @@ -22,70 +22,6 @@ void log_nvml_error(nvmlReturn_t rc, char* gpuid, const char* id); #ifdef NVML_API -/* - * get_non_nvml_device() - * - * hwloc's method to identify a GPU failed, so use ours. Theirs fails a lot. - * - * @param topology - the topology that the device should fall under - * @param device - the nvml device handle - * @param identified - a list of the previously identified devices to make sure - * we aren't storing the same one multiple times. - * @return an hwloc obj pointing to the GPU - */ - -hwloc_obj_t Machine::get_non_nvml_device( - - hwloc_topology_t topology, - nvmlDevice_t device, - std::set &identified) - - { - hwloc_obj_t osdev; - nvmlReturn_t nvres; - nvmlPciInfo_t pci; - - if (!hwloc_topology_is_thissystem(topology)) - { - errno = EINVAL; - return NULL; - } - - nvres = nvmlDeviceGetPciInfo(device, &pci); - if (NVML_SUCCESS != nvres) - return(NULL); - - osdev = NULL; - while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) - { - hwloc_obj_t pcidev = osdev->parent; - if ((strncmp(osdev->name, "card", 4)) && - (strncmp(osdev->name, "controlD", 8))) - continue; - - if ((pcidev != NULL) && - (pcidev->type == HWLOC_OBJ_PCI_DEVICE) && - (pcidev->attr->pcidev.domain == pci.domain) && - (pcidev->attr->pcidev.bus == pci.bus) && - (pcidev->attr->pcidev.dev == pci.device) && - (pcidev->attr->pcidev.func == 0)) - return(osdev); - } - - // We haven't found it yet, but there are GPUs and hwloc knows it. Just match on type. - osdev = NULL; - while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) - { - if ((osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_GPU) && - (identified.find(osdev) == identified.end())) - return(osdev); - } - - return(NULL); - } // END get_non_nvml_device() - - - int Machine::initializeNVIDIADevices( hwloc_obj_t machine_obj, @@ -93,12 +29,17 @@ int Machine::initializeNVIDIADevices( { nvmlReturn_t rc; + int rc_init = PBSE_NONE; + unsigned int device_count; /* Initialize the NVML handle. * * nvmlInit should be called once before invoking any other methods in the NVML library. * A reference count of the number of initializations is maintained. Shutdown only occurs * when the reference count reaches zero. + * + * This routine does not use hwloc since as of <= 1.11.7 it is known to fail to identifiy NVIDIA devices + * on some systems. * */ rc = nvmlInit(); if (rc != NVML_SUCCESS && rc != NVML_ERROR_ALREADY_INITIALIZED) @@ -107,10 +48,6 @@ int Machine::initializeNVIDIADevices( return(PBSE_NONE); } - std::vector unfound_ids; - unsigned int device_count = 0; - unsigned int found_devices = 0; - /* Get the device count. */ rc = nvmlDeviceGetCount(&device_count); if (rc == NVML_SUCCESS) @@ -121,72 +58,14 @@ int Machine::initializeNVIDIADevices( /* Get the nvml device handle at each index */ for (unsigned int idx = 0; idx < device_count; idx++) { - rc = nvmlDeviceGetHandleByIndex(idx, &gpu); - - if (rc != NVML_SUCCESS) - { - /* TODO: get gpuid from nvmlDevice_t struct */ - log_nvml_error(rc, NULL, __func__); - } - - /* Use the hwloc library to determine device locality */ - hwloc_obj_t gpu_obj; - hwloc_obj_t ancestor_obj; - int is_in_tree; - - gpu_obj = hwloc_nvml_get_device_osdev(topology, gpu); - if (gpu_obj == NULL) - { - // This was not an nvml device. We'll look for a different GPU - many aren't recognized, - // including k80s. - gpu_obj = this->get_non_nvml_device(topology, gpu, identified); - if (gpu_obj == NULL) - { - unfound_ids.push_back(idx); - continue; - } - } - - identified.insert(gpu_obj); - found_devices++; - - /* The ancestor was not a numa chip. Is it the machine? */ - ancestor_obj = hwloc_get_ancestor_obj_by_type(topology, HWLOC_OBJ_MACHINE, gpu_obj); - if (ancestor_obj != NULL) - { - char buf[MAXLINE]; - rc = nvmlDeviceGetName(gpu, buf, sizeof(buf)); - - PCI_Device new_device; - - new_device.initializePCIDevice(gpu_obj, idx, topology); - - store_device_on_appropriate_chip(new_device, false); - - // hwloc sees the K80 as a single device. We want to display two to stay in sync - // with NVML, so store an extra for each K80 - if (!strncmp(buf, "Tesla K80", 9)) - { - new_device.setId(new_device.get_id() + 1); - store_device_on_appropriate_chip(new_device, false); - found_devices++; - idx++; - } - } - } - - if (found_devices != device_count) - { - sprintf(log_buffer, "NVML reports %u devices, but we only found %u. Placement will be approximated.", - device_count, found_devices); - log_err(-1, __func__, log_buffer); + PCI_Device new_device; - for (size_t i = 0; i < unfound_ids.size(); i++) + if (new_device.initializePCIDevice(NULL, idx, topology) != PBSE_NONE) { - PCI_Device new_device; - new_device.initializePCIDevice(NULL, unfound_ids[i], topology); - store_device_on_appropriate_chip(new_device, true); + rc_init = -1; + break; } + store_device_on_appropriate_chip(new_device, false); } } else @@ -207,7 +86,7 @@ int Machine::initializeNVIDIADevices( log_nvml_error(rc, NULL, __func__); } - return(PBSE_NONE); + return(rc_init); } #endif @@ -223,7 +102,6 @@ int Chip::initializeMICDevices(hwloc_obj_t chip_obj, hwloc_topology_t topology) { hwloc_obj_t mic_obj; hwloc_obj_t ancestor_obj; - int is_in_tree; mic_obj = hwloc_intel_mic_get_device_osdev_by_index(topology, idx); if (mic_obj == NULL) @@ -289,16 +167,33 @@ void PCI_Device::initializeMic( #ifdef NVIDIA_GPUS +/* + * due to an hwloc limitation, it can't be depended on to report the cpu information about an nvml device + * so we work around that by getting the cpulist information in a different way (sysfs) + */ + void PCI_Device::initializeGpu( - int idx, - hwloc_topology_t topology) + int idx) { int rc; nvmlDevice_t gpu_device; - + nvmlPciInfo_t pci; + char cpulist_path[PATH_MAX]; + FILE *fp; + char *p; + id = idx; + + this->type = GPU; + + // look up nearest cpuset of device + // Can do this one of 3 ways: + // 1) use hwloc (broken in 1.11--doesn't id some gpu devices) + // 2) use nvmlDeviceGetCpuAffinity() (need to convert bitmap to string) + // 3) look up in sysfs (no conversion needed so use this method) + rc = nvmlDeviceGetHandleByIndex(idx, &gpu_device); if (rc != NVML_SUCCESS) { @@ -307,28 +202,38 @@ void PCI_Device::initializeGpu( buf = "nvmlDeviceGetHandleByIndex failed for nvidia gpus"; buf = buf + name.c_str(); log_err(-1, __func__, buf.c_str()); + return; } - else - { - nearest_cpuset = hwloc_bitmap_alloc(); - if (nearest_cpuset != NULL) - { - rc = hwloc_nvml_get_device_cpuset(topology, gpu_device, nearest_cpuset); - if (rc != 0) - { - string buf; - buf = "could not get cpuset of "; - buf = buf + name.c_str(); - log_err(-1, __func__, buf.c_str()); - } + // get the PCI info from the NVML identified device + rc = nvmlDeviceGetPciInfo(gpu_device, &pci); + if (rc != NVML_SUCCESS) + { + snprintf(log_buffer, sizeof(log_buffer), "nvmlDeviceGetPciInfo failed with %d for index %d", + rc, idx); + log_err(-1, __func__, log_buffer); + return; + } + + // build path to cpulist for this PCI device + snprintf(cpulist_path, sizeof(cpulist_path), "/sys/bus/pci/devices/%s/local_cpulist", + pci.busId); - hwloc_bitmap_list_snprintf(cpuset_string, MAX_CPUSET_SIZE, nearest_cpuset); - } + // open cpulist + if ((fp = fopen(cpulist_path, "r")) == NULL) + { + snprintf(log_buffer, sizeof(log_buffer), "could not open %s", cpulist_path); + log_err(-1, __func__, log_buffer); + return; } - this->type = GPU; + // read cpulist + fgets(cpuset_string, MAX_CPUSET_SIZE, fp); + fclose(fp); + // delete the trailing newline + if ((p = strchr(cpuset_string, '\n')) != NULL) + *p = '\0'; } #endif diff --git a/src/server/pbsd_main.c b/src/server/pbsd_main.c index c6db8b05b0..b927d254a9 100644 --- a/src/server/pbsd_main.c +++ b/src/server/pbsd_main.c @@ -3048,7 +3048,7 @@ int unlock_sv_qs_mutex(pthread_mutex_t *sv_qs_mutex, const char *msg_string) // Stub out some functions for NUMA #ifdef NVIDIA_GPUS int Machine::initializeNVIDIADevices(hwloc_obj_t, hwloc_topology_t) {return(0);} -extern void PCI_Device::initializeGpu(int idx, hwloc_topology_t topology) {} +extern void PCI_Device::initializeGpu(int idx) {} #endif #ifdef MIC int Chip::initializeMICDevices(hwloc_obj_t chip_obj, hwloc_topology_t topology) {return(0);} From e142e2e1ecdb259884af3e11058159accff6ade5 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 27 Oct 2017 15:19:41 -0600 Subject: [PATCH 304/312] TRQ-3978 - correct buffer size when getting NVIDIA device name --- src/lib/Libutils/numa_pci_device.cpp | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/lib/Libutils/numa_pci_device.cpp b/src/lib/Libutils/numa_pci_device.cpp index f53f1e9f71..45d8d4feda 100644 --- a/src/lib/Libutils/numa_pci_device.cpp +++ b/src/lib/Libutils/numa_pci_device.cpp @@ -75,29 +75,19 @@ int PCI_Device::initializePCIDevice(hwloc_obj_t device_obj, int idx, hwloc_topol // use NVML library to look up info about the device nvmlDevice_t gpu; - char *buf; + char buf[NVML_DEVICE_NAME_BUFFER_SIZE]; // get the device handle so the name can be looked up if (nvmlDeviceGetHandleByIndex(idx, &gpu) != NVML_SUCCESS) return(-1); - // allocate space for the name - if ((buf = (char *)malloc(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE)) == NULL) - return(-1); - // look up the name - if (nvmlDeviceGetName(gpu, buf, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE) != NVML_SUCCESS) - { - free(buf); + if (nvmlDeviceGetName(gpu, buf, sizeof(buf)) != NVML_SUCCESS) return(-1); - } // copy the name name = buf; - // free the buffer - free(buf); - // copy the id id = idx; #else From a9193e9f9f9f4ca6211e7933cec9716794e82dde Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Mon, 20 Nov 2017 14:56:15 -0700 Subject: [PATCH 305/312] TRQ-4138 - maintain proper user job counts when submitting jobs with failed dependencies --- src/server/svr_jobfunc.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index 68aac16318..8938c2ce7b 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -696,7 +696,17 @@ int svr_enquejob( if (rc == PBSE_JOBNOTFOUND) return(rc); else if (rc != PBSE_NONE) + { + // decrement queued job counts + + pque->qu_numjobs--; + pque->qu_njstate[pjob->ji_qs.ji_state]--; + + decrement_queued_jobs(pque->qu_uih, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + decrement_queued_jobs(&users, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + rc = PBSE_BADDEPEND; + } } return(rc); From 8b11a5308d69111127d75b26991c365f749a0ab6 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 1 Dec 2017 14:47:42 -0700 Subject: [PATCH 306/312] TRQ-4115 - fix pbs_server deadlock --- src/server/req_jobobit.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/req_jobobit.c b/src/server/req_jobobit.c index 3a5122c7f4..583a37f801 100644 --- a/src/server/req_jobobit.c +++ b/src/server/req_jobobit.c @@ -3424,6 +3424,9 @@ bool is_job_finished( // Must be a version 6.1.0 node or higher for the mom to have cleaned up the job if (pnode->get_version() >= 610) { + // unlock the node first + node_mutex.unlock(); + /* see if job has any dependencies */ if (pjob->ji_wattr[JOB_ATR_depend].at_flags & ATR_VFLAG_SET) { @@ -3434,8 +3437,6 @@ bool is_job_finished( } } - node_mutex.unlock(); - rel_resc(pjob); svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE); handle_complete_first_time(pjob); From f32dfc71f6b9e1de2d3512045e93e1adce03bda5 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 2 Feb 2018 08:57:58 -0700 Subject: [PATCH 307/312] TRQ-4161 - make additional check before deleting job --- src/server/req_delete.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/server/req_delete.c b/src/server/req_delete.c index 80892c868e..8143cd3f6b 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -875,6 +875,15 @@ int delete_all_work( while ((pjob = next_job(&alljobs, iter)) != NULL) { + // allowed to delete? + chk_job_req_permissions(&pjob, preq); + + if (pjob == NULL) + { + // not allowed + continue; + } + if ((pjob->ji_arraystructid[0] != '\0') && (pjob->ji_is_array_template == false)) { From 41d0ee60447bc303ab8dc9e88978b47e93d22e90 Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Fri, 2 Feb 2018 10:47:43 -0700 Subject: [PATCH 308/312] TRQ-4161 - small optimization to original commit --- src/server/req_delete.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/server/req_delete.c b/src/server/req_delete.c index 8143cd3f6b..f558dfd240 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -875,18 +875,18 @@ int delete_all_work( while ((pjob = next_job(&alljobs, iter)) != NULL) { - // allowed to delete? - chk_job_req_permissions(&pjob, preq); - - if (pjob == NULL) - { - // not allowed - continue; - } - if ((pjob->ji_arraystructid[0] != '\0') && (pjob->ji_is_array_template == false)) { + // allowed to delete? + chk_job_req_permissions(&pjob, preq); + + if (pjob == NULL) + { + // not allowed + continue; + } + // Mark this array as being deleted if it hasn't yet been marked if (marked_arrays.find(pjob->ji_arraystructid) == marked_arrays.end()) { From 95f1eac0266646d336cb4ae48456ce7263dbf3cf Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Wed, 7 Feb 2018 16:31:15 -0700 Subject: [PATCH 309/312] TRQ-4161 - fix problem with original patch --- src/server/req_delete.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/server/req_delete.c b/src/server/req_delete.c index f558dfd240..2ac7adf8d7 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -167,6 +167,7 @@ extern void set_resc_assigned(job *, enum batch_op); extern job *chk_job_request(char *, struct batch_request *); extern struct batch_request *cpy_stage(struct batch_request *, job *, enum job_atr, int); extern int svr_chk_owner(struct batch_request *, job *); +extern int svr_authorize_jobreq(struct batch_request *, job *); void chk_job_req_permissions(job **,struct batch_request *); void on_job_exit_task(struct work_task *); void remove_stagein(job **pjob_ptr); @@ -879,9 +880,7 @@ int delete_all_work( (pjob->ji_is_array_template == false)) { // allowed to delete? - chk_job_req_permissions(&pjob, preq); - - if (pjob == NULL) + if (svr_authorize_jobreq(preq, pjob) == -1) { // not allowed continue; From 62ae99b7290f4cfdbf0c841da9857932f779b76f Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 8 Feb 2018 14:02:40 -0700 Subject: [PATCH 310/312] Fix unit test. --- src/test/req_delete/scaffolding.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/req_delete/scaffolding.c b/src/test/req_delete/scaffolding.c index 229bc6a454..eb6f284e4c 100644 --- a/src/test/req_delete/scaffolding.c +++ b/src/test/req_delete/scaffolding.c @@ -567,3 +567,7 @@ pbs_net_t get_hostaddr( return(0); } +int svr_authorize_jobreq(batch_request *brp, job *pjob) + { + return(-1); + } From 661e092552de43a785c15d39a3634a541d86898e Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 8 Feb 2018 14:53:07 -0700 Subject: [PATCH 311/312] TRQ-4161 - fix hang condition --- src/server/req_delete.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/server/req_delete.c b/src/server/req_delete.c index 2ac7adf8d7..d04ee08a0b 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -877,15 +877,9 @@ int delete_all_work( while ((pjob = next_job(&alljobs, iter)) != NULL) { if ((pjob->ji_arraystructid[0] != '\0') && - (pjob->ji_is_array_template == false)) + (pjob->ji_is_array_template == false) && + (svr_authorize_jobreq(preq, pjob) == 0)) { - // allowed to delete? - if (svr_authorize_jobreq(preq, pjob) == -1) - { - // not allowed - continue; - } - // Mark this array as being deleted if it hasn't yet been marked if (marked_arrays.find(pjob->ji_arraystructid) == marked_arrays.end()) { From f242894c904ede8aa3ba0c3fa3929c00275ed67e Mon Sep 17 00:00:00 2001 From: Chad Vizino Date: Thu, 8 Feb 2018 14:53:07 -0700 Subject: [PATCH 312/312] TRQ-4161 - fix hang condition --- src/cmds/qsub_functions.c | 44 ++++------------- src/include/utils.h | 1 + src/lib/Libutils/lib_utils.h | 1 + src/lib/Libutils/u_groups.c | 93 ++++++++++++++++++++++++++++++++++++ src/lib/Libutils/u_wrapper.c | 10 +--- src/lib/Libutils/u_wrapper.h | 42 ++++++++++++++++ src/server/req_delete.c | 10 +--- src/server/svr_jobfunc.c | 17 +------ 8 files changed, 150 insertions(+), 68 deletions(-) create mode 100644 src/lib/Libutils/u_wrapper.h diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index eabbf66bde..9843557415 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -2626,8 +2626,6 @@ void interactive( } /* END interactive() */ - - int validate_group_list( char *glist) @@ -2635,14 +2633,11 @@ int validate_group_list( { /* check each group to determine if it is a valid group that the user can be a part of. * group list is of the form group[@host][,group[@host]...] */ - char *buf = NULL; char *groups = strdup(glist); const char *delims = ","; - char *tmp_group = strtok(groups, delims); + char *tmp_group = strtok(groups, delims); char *at; char *u_name; - char **pmem; - struct group *grent; struct passwd *pwent; if ((pwent = getpwuid(getuid())) == NULL) @@ -2657,46 +2652,23 @@ int validate_group_list( { if ((at = strchr(tmp_group,'@')) != NULL) *at = '\0'; - - if ((grent = getgrnam_ext(&buf, tmp_group)) == NULL) - { - free(groups); - return(FALSE); - } - - pmem = grent->gr_mem; - free_grname(grent, buf); - - if (pmem == NULL) - { - free(groups); - return(FALSE); - } - - while (*pmem != NULL) - { - if (!strcmp(*pmem,u_name)) - break; - - pmem++; - } - if (*pmem == NULL) - { - /* match not found */ + if (is_group_member(u_name, tmp_group) == FALSE) + { free(groups); - return(FALSE); - } - + return(false); + } + tmp_group = strtok(NULL,delims); } free(groups); - return(TRUE); + return(true); } + bool came_from_moab( const char *src, diff --git a/src/include/utils.h b/src/include/utils.h index 3b2aa80314..e1a3fbd815 100644 --- a/src/include/utils.h +++ b/src/include/utils.h @@ -145,6 +145,7 @@ extern char *OriginalPath; /* group functions in u_groups.c */ extern void free_grname(struct group *, char *); extern void free_pwnam(struct passwd *pwdp, char *buf); +extern bool is_group_member(char *user_name, char *group_name); extern struct group *getgrnam_ext (char **, char *); extern struct group *getgrgid_ext (char **, gid_t); diff --git a/src/lib/Libutils/lib_utils.h b/src/lib/Libutils/lib_utils.h index 22688f14d3..a20d77fd52 100644 --- a/src/lib/Libutils/lib_utils.h +++ b/src/lib/Libutils/lib_utils.h @@ -41,6 +41,7 @@ int MXMLFromString(mxml_t **EP, char *XMLString, char **Tail, char *EMsg, /* u_groups.c */ void free_grname(struct group *, char *); +bool is_group_member(char *user_name, char *group_name); struct group * getgrnam_ext(char **user_buf, char * grp_name ); struct group * getgrgid_ext(char **user_buf, gid_t grp_id ); diff --git a/src/lib/Libutils/u_groups.c b/src/lib/Libutils/u_groups.c index 5d64622ef0..0708479452 100644 --- a/src/lib/Libutils/u_groups.c +++ b/src/lib/Libutils/u_groups.c @@ -1,6 +1,9 @@ #include "license_pbs.h" /* See here for the software license */ #include +#include +#include #include "utils.h" +#include "u_wrapper.h" void free_grname( @@ -23,6 +26,96 @@ void free_grname( } + /** + * is_group_member - checks to see if user_name is a member of group_name + * + * @param user_name (I) - name of user being checked for group membership + * @param group_name (I) - group name being checked for user membership + * @return true if user_name is a member of group_name. Otherwise return false + * + **/ + bool is_group_member(char *user_name, char *group_name) + { + int j, ngroups; + gid_t *groups; + struct passwd *pw = NULL; + struct group *gr = NULL; + char *user_buffer = NULL; + char log_buf[LOCAL_LOG_BUF_SIZE]; + + ngroups = 0; + + // You can malloc a 0 byte buffer + groups = (gid_t *)malloc(ngroups * sizeof(gid_t)); + if (groups == NULL) + { + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, "failed to allocate memory"); + return(false); + } + + pw = getpwnam_wrapper(&user_buffer, user_name); + if (pw == NULL) + { + free(groups); + if (user_buffer != NULL) + { + free(user_buffer); + } + sprintf(log_buf, "user %s not found.", user_name); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); + return(false); + } + + // The first call to getgrouplist is intended to get the number of + // groups for user_name. It will fail but set ngroups to the right number + if (getgrouplist(user_name, pw->pw_gid, groups, &ngroups) == -1) + { + free(groups); + if (ngroups > 0) + { + groups = (gid_t*)malloc(ngroups * sizeof(gid_t)); + if (groups == NULL) + { + free(pw); + free(user_buffer); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, "failed to allocate memory for getgrouplist"); + return(false); + } + if (getgrouplist(user_name, pw->pw_gid, groups, &ngroups) == -1) + { + sprintf(log_buf, " Call to getgrouplist failed: %s", strerror(errno)); + free(pw); + free(user_buffer); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); + return(false); + } + } + } + + for (j = 0; j < ngroups; j++) + { + gr = getgrgid(groups[j]); + if (gr == NULL) + { + continue; + } + + if (!strcasecmp(gr->gr_name, group_name)) + { + free(pw); + free(user_buffer); + free(groups); + return(true); + } + } + + free(pw); + free(user_buffer); + free(groups); + return(false); + } + + /** * getgrnam_ext - first calls getgrnam, and if this call doesn't return * anything, then it checks if the name is actually a group id by calling getgrgid diff --git a/src/lib/Libutils/u_wrapper.c b/src/lib/Libutils/u_wrapper.c index 72d103d769..d419d2b51f 100644 --- a/src/lib/Libutils/u_wrapper.c +++ b/src/lib/Libutils/u_wrapper.c @@ -1,14 +1,6 @@ #include "license_pbs.h" /* See here for the software license */ -#include /* sprintf */ -#include /* calloc */ -#include -#include -#include -#include -#include -#include - +#include "u_wrapper.h" #include "log.h" #include "pbs_error.h" diff --git a/src/lib/Libutils/u_wrapper.h b/src/lib/Libutils/u_wrapper.h new file mode 100644 index 0000000000..5bf2659f01 --- /dev/null +++ b/src/lib/Libutils/u_wrapper.h @@ -0,0 +1,42 @@ +#include "license_pbs.h" /* See here for the software license */ + +#include /* sprintf */ +#include /* calloc */ +#include +#include +#include +#include +#include +#include + + +/** + * getpwnam_wrapper - Get the passwd structure of a user + * + * getpwnam_wrapper allocates all the needed memory and makes a call to getpwnam_r. + * + * @param user_buffer - A pointer to the buffer containing the string fields pointed to by the passwd structure + * @param user_name - The name of the user for which user information is being retrieved. + * + * @returns a struct passwd pointer. The string fields in the passwd structure point to strings in user_buffer + */ +struct passwd *getpwnam_wrapper( + char **user_buffer, /* getpwnam_r uses a buffer which must be freed later. This is a pointer to that buffer */ + const char *user_name + ); + +struct passwd *getpwuid_wrapper( + char **user_buf, + uid_t uid); + +int rmdir_ext( + const char *dir, + int retry_limit); + +int unlink_ext( + const char *filename, + int retry_limit); + +int mkdir_wrapper( + const char *pathname, + mode_t mode); diff --git a/src/server/req_delete.c b/src/server/req_delete.c index 2ac7adf8d7..d04ee08a0b 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -877,15 +877,9 @@ int delete_all_work( while ((pjob = next_job(&alljobs, iter)) != NULL) { if ((pjob->ji_arraystructid[0] != '\0') && - (pjob->ji_is_array_template == false)) + (pjob->ji_is_array_template == false) && + (svr_authorize_jobreq(preq, pjob) == 0)) { - // allowed to delete? - if (svr_authorize_jobreq(preq, pjob) == -1) - { - // not allowed - continue; - } - // Mark this array as being deleted if it hasn't yet been marked if (marked_arrays.find(pjob->ji_arraystructid) == marked_arrays.end()) { diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index 8938c2ce7b..cfdd98f609 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -2286,9 +2286,7 @@ static int check_queue_group_ACL( { /* check group acl against all accessible groups */ - struct group *grp; int i = 0; - int j = 0; char uname[PBS_MAXUSER + 1]; @@ -2298,26 +2296,15 @@ static int check_queue_group_ACL( pas = pque->qu_attr[QA_ATR_AclGroup].at_val.at_arst; + rc = 0; for (i = 0; pas != NULL && i < pas->as_usedptr;i++) { - char *buf = NULL; - if ((grp = getgrnam_ext(&buf, pas->as_string[i])) == NULL) - continue; - - for (j = 0;grp->gr_mem[j] != NULL;j++) + if (is_group_member(uname, pas->as_string[i]) == true) { - if (!strcmp(grp->gr_mem[j], uname)) - { rc = 1; - break; - } } - free_grname(grp, buf); - - if (rc == 1) - break; } } /* END if (rc == 0) && slpygrp && ...) */