merge with david

.gitignore

@@ -4,3 +4,4 @@
 \#*\#
 LSFJOB*
 *RunII*sh
+*.db

README.md

@@ -1,7 +1,7 @@
 Scripts for McM
 ===============
 
-Author: David G. Sheffield (Rutgers)
+Author: David G. Sheffield (Rutgers), extension of getRequests.py by Luca Perrozzi (ETHZ)
 
 Scripts for creating and updating requests in McM. See https://twiki.cern.ch/twiki/bin/viewauth/CMS/PdmVMcMScript for more information on scripts.
 
@@ -18,16 +18,27 @@ Scripts
 * [**getMcMTestScript.sh**](#usage-of-getmcmtestscriptsh) Get test script from McM with some modifications.
 * [**getTimeSize.sh**](#usage-of-gettimesizesh) Extract time/size per event from cmsRun job report.
 
+Installation
+------------
+
+Once you've created a clone of this repository run the command
+
+```
+git update-index --assume-unchanged mcmscripts_config.py
+```
+
+to avoid committing any changes to your personal configuration file. Then modify the default values in mcmscripts_config.py to use your physics working group and username as default.
+
 Get cookies
 -----------
 
 Some scripts require getting a CERN SSO cookie before using them. They can be obtained for the production instance of McM with
 
-`cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess`
+`cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb`
 
 and for the dev/test instance with
 
-`cern-get-sso-cookie -u https://cms-pdmv-dev.cern.ch/mcm/ -o ~/private/dev-cookie.txt --krb --reprocess`
+`cern-get-sso-cookie -u https://cms-pdmv-dev.cern.ch/mcm/ -o ~/private/dev-cookie.txt --krb`
 
 Run these commands before setting up a CMSSW environment. If these to not work, try to run [getCookie.sh](getCookie.sh).
 
@@ -55,7 +66,7 @@ Usage of manageRequests.py
 
 ### Setup PWG
 
-The default PWG in manageRequests.py is set to XXX. To modify your default PWG, change the variable `defaultPWG` on line 23. Alternatively, you can include the flag `-p your_PWG`.
+The default PWG in manageRequests.py is set by `mcmscripts_config.py`. To set it to your group, change the configuration file. Alternatively, you can include the flag `-p your_PWG`.
 
 ### Creating new requests
 
@@ -105,6 +116,7 @@ Information for requests is provided in a CSV file. The script reads the first l
 * Gridpack location
 * Gridpack cards URL
 * Notes
+* McM tag
 
 It will also recognize some alternative names. If there is a field title that the script does not recognize, it will complain. The script will ignore columns with the headers "JobId", "Local gridpack location", and "Local LHE" as they are used to supply information to other scripts but do not contain information for McM.
 
@@ -135,6 +147,10 @@ The script can sort for only requests with time per event and size per event set
 
 To obtain a list of chained requests from wmLHE requests you can use the `-c` flag.
 
+NEW: a new funcionality has been added to dump much more information (with COLORS!), to be launched with the flag '--listattr'. The default is `-f 0` (Dataset name, Extension, Number of Completed/Total events). The level of verbosity can be increased to `-f 1` (Status, Time Event, CMSSW Release, Priority), `-f 2` (Cross Section, Filter efficiency, Matching efficiency, Tags, Generators, Name of Fragment, Notes), `-f 3` (Last Updater Name and Email, McM View and Edit Links), `-f 4` (Member of the chains including prepIds of the chained requests and direct McM chain link), and `-f 5` (Fragment code). Example:
+
+`python getRequests.py -listattr 5 "actor=perrozzi&member_of_campaign=*GS*&status=new"`
+
 Usage of testRequests.py
 ------------------------
 

addTags.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+
+################################
+#
+# addTags.py
+#
+#  Script to add tags to old requests.
+#
+#  author: David G. Sheffield (Rutgers)
+#
+################################
+
+import sys
+import os.path
+import argparse
+import time
+import mcmscripts_config
+sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
+from rest import * # Load class to access McM
+
+
+def getArguments():
+    parser = argparse.ArgumentParser(
+        description='Tag old requests.')
+
+    # Command line flags
+    parser.add_argument('ids', metavar='PrepIDs',
+                        help='List of PrepIDs to tag copies of.')
+    parser.add_argument('tag', metavar='Tag')
+
+    args_ = parser.parse_args()
+    return args_
+
+
+def fillIDRange(pwg, campaign, first, last):
+    first = int(first)
+    last = int(last)
+    chains = []
+    if first > last:
+        print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format(
+            pwg, campaign, first, last)
+        sys.exit(1)
+
+    for number in range(first, last+1):
+        chains.append("{0}-{1}-{2:05d}".format(pwg, campaign, number))
+    return chains
+
+
+def parseIDList(compactList):
+    splitList = compactList.split(',')
+    chains = []
+    for subList in splitList:
+        splitSubList = subList.split('-')
+        if len(splitSubList) == 3:
+            chains.append(subList)
+        elif len(splitSubList) == 4:
+            chains += fillIDRange(splitSubList[0], splitSubList[1],
+                                  splitSubList[2], splitSubList[3])
+        elif len(splitSubList) == 6:
+            if splitSubList[0] != splitSubList[3]:
+                print "Error: PrepID range must be for the same PWG."
+                sys.exit(1)
+            if splitSubList[1] != splitSubList[4]:
+                print "Error: PrepID range must be for the same chained campaign."
+                sys.exit(1)
+            chains += fillIDRange(splitSubList[0], splitSubList[1],
+                                  splitSubList[2], splitSubList[5])
+        else:
+            print "Error: Poorly formed PrepID list."
+            sys.exit(1)
+    return chains
+
+
+def modifyRequest(mcm, req, tag):
+    for existing in req['tags']:
+        if existing.startswith("EXO"):
+            print "  \033[0;33m{0} has already been tagged with {1}\033[0;m".format(req['prepid'], existing)
+            return
+    req['tags'] += [tag]
+    answer = mcm.updateA('requests', req) # Update request
+    if answer['results']:
+        print "  \033[0;32m{0} tagged {1}\033[0;m".format(req['prepid'], tag)
+    else:
+        print "  \033[0;32m{0} not tagged\033[0;m".format(req['prepid'])
+    return
+
+
+def tagRequests(prepids, tag):
+    mcm = restful(dev=False)
+    campaigns = ['RunIISummer15GS', 'RunIIFall15DR76', 'RunIIFall15MiniAODv1',
+                 'RunIIFall15MiniAODv2']
+
+    num = len(prepids)
+    counter = 0
+
+    for prepid in prepids:
+        counter += 1
+        time.sleep(1.0)
+        req = mcm.getA('requests', prepid)
+        print "{0}/{1} {2} {3}".format(counter, num, prepid, req['dataset_name'])
+        if req['member_of_campaign'] == "RunIIWinter15wmLHE"\
+                or req['member_of_campaign'] == "RunIIWinter15pLHE":
+            modifyRequest(mcm, req, tag)
+        dataset_name = req['dataset_name']
+        for campaign in campaigns:
+            query_string = "dataset_name={0}&member_of_campaign={1}".format(
+                dataset_name, campaign)
+            failed_to_get = True
+            for tries in range(3):
+                time.sleep(0.5)
+                req_list = mcm.getA('requests', query=query_string)
+                if req_list is not None:
+                    failed_to_get = False
+                    break
+            if failed_to_get:
+                print "  \033[0;31mCould not find {0} in {1}\033[0;m".format(
+                    dataset_name, campaign)
+                continue
+            if len(req_list) > 1:
+                print "  \033[0;31m{0} has too many requests in {1}\033[0;m".format(
+                    dataset_name, campaign)
+                continue
+            if len(req_list) == 0:
+                print "  \033[0;31m{0} does not exist in {1}\033[0;m".format(
+                    dataset_name, campaign)
+                continue
+            req = req_list[0]
+            modifyRequest(mcm, req, tag)
+    return
+
+
+def main():
+    args = getArguments()
+    prepids = parseIDList(args.ids)
+    print "Tagging {0} base requests".format(len(prepids))
+    tagRequests(prepids, args.tag)
+
+
+if __name__ == '__main__':
+    main()

batchCloneRequests.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+
+################################
+#
+# batchCloneRequests.py
+#
+#  Script to clone multiple McM requests.
+#
+#  author: David G. Sheffield (Rutgers)
+#
+################################
+
+import sys
+import os.path
+import argparse
+import time
+import mcmscripts_config
+sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
+from rest import * # Load class to access McM
+
+
+def getArguments():
+    parser = argparse.ArgumentParser(
+        description='Clone multiple McM requests.')
+
+    # Command line flags
+    parser.add_argument('-i', '--ids', dest='ids', help=
+                        'List of PrepIDs to be cloned.')
+    parser.add_argument('-c', '--campaign', action='store', dest='campaign',
+                        metavar='name', help='Set member_of_campaign.')
+
+    args_ = parser.parse_args()
+    return args_
+
+
+def fillIDRange(pwg, campaign, first, last):
+    first = int(first)
+    last = int(last)
+    requests = []
+    if first > last:
+        print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format(
+            pwg, campaign, first, last)
+        sys.exit(1)
+
+    for number in range(first, last+1):
+        requests.append("{0}-{1}-{2:05d}".format(pwg, campaign, number))
+    return requests
+
+
+def parseIDList(compactList):
+    print compactList
+    splitList = compactList.split(',')
+    requests = []
+    for subList in splitList:
+        splitSubList = subList.split('-')
+        if len(splitSubList) == 3:
+            requests.append(subList)
+        elif len(splitSubList) == 4:
+            requests += fillIDRange(splitSubList[0], splitSubList[1],
+                                    splitSubList[2], splitSubList[3])
+        elif len(splitSubList) == 6:
+            if splitSubList[0] != splitSubList[3]:
+                print "Error: PrepID range must be for the same PWG."
+                sys.exit(1)
+            if splitSubList[1] != splitSubList[4]:
+                print "Error: PrepID range must be for the same chained campaign."
+                sys.exit(1)
+            requests += fillIDRange(splitSubList[0], splitSubList[1],
+                                    splitSubList[2], splitSubList[5])
+        else:
+            print "Error: Poorly formed PrepID list."
+            sys.exit(1)
+    return requests
+
+
+def cloneRequests(requests, campaign):
+    # Create new requests be cloning an old one based on PrepId
+    useDev = False
+    mcm = restful(dev=useDev) # Get McM connection
+
+    print "Adding {0} requests to McM using clone.".format(len(requests))
+    for PrepID in requests:
+        clone_req = mcm.getA('requests', PrepID) # Get request to clone
+        clone_req['member_of_campaign'] = campaign
+        answer = mcm.clone(PrepID, clone_req) # Clone request
+        if answer['results']:
+            print "\033[0;32m{0} cloned from {1}\033[0;m".format(
+                answer['prepid'], PrepID)
+        else:
+            print "\033[0;31m{0} failed to be cloned to {1}\033[0;m".format(
+                PrepID, campaign)
+        time.sleep(0.5)
+
+
+def main():
+    args = getArguments() # Setup flags and get arguments
+
+    requests = parseIDList(args.ids)
+    cloneRequests(requests, args.campaign)
+
+
+if __name__ == '__main__':
+    main()

checkRequests.py

@@ -36,9 +36,8 @@ def fillIDRange(pwg, campaign, first, last):
     last = int(last)
     requests = []
     if first > last:
-        print "Error: PrepID range out of order. %s-%s-%05d > %s-%s-%05d" % (
-            pwg, campaign, first, pwg, campaign, last)
-        print "Exiting with status 4."
+        print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format(
+            pwg, campaign, first, last)
         sys.exit(4)
 
     for number in range(first, last+1):
@@ -59,17 +58,14 @@ def parseIDList(compactList):
         elif len(splitSubList) == 6:
             if splitSubList[0] != splitSubList[3]:
                 print "Error: PrepID range must be for the same PWG."
-                print "Exiting with status 4"
                 sys.exit(4)
             if splitSubList[1] != splitSubList[4]:
                 print "Error: PrepID range must be for the same campaign."
-                print "Exiting with status 4"
                 sys.exit(4)
             requests = requests + fillIDRange(splitSubList[0], splitSubList[1],
                                               splitSubList[2], splitSubList[5])
         else:
             print "Error: Poorly formed PrepID list."
-            print "Exiting with status 3."
             sys.exit(3)
     return requests