diff --git a/.gitignore b/.gitignore index 9dc8ee7..1340512 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ \#*\# LSFJOB* *RunII*sh +*.db diff --git a/README.md b/README.md index 1bbb523..71cddf7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Scripts for McM =============== -Author: David G. Sheffield (Rutgers) +Author: David G. Sheffield (Rutgers), extension of getRequests.py by Luca Perrozzi (ETHZ) Scripts for creating and updating requests in McM. See https://twiki.cern.ch/twiki/bin/viewauth/CMS/PdmVMcMScript for more information on scripts. @@ -18,16 +18,27 @@ Scripts * [**getMcMTestScript.sh**](#usage-of-getmcmtestscriptsh) Get test script from McM with some modifications. * [**getTimeSize.sh**](#usage-of-gettimesizesh) Extract time/size per event from cmsRun job report. +Installation +------------ + +Once you've created a clone of this repository run the command + +``` +git update-index --assume-unchanged mcmscripts_config.py +``` + +to avoid committing any changes to your personal configuration file. Then modify the default values in mcmscripts_config.py to use your physics working group and username as default. + Get cookies ----------- Some scripts require getting a CERN SSO cookie before using them. They can be obtained for the production instance of McM with -`cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess` +`cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb` and for the dev/test instance with -`cern-get-sso-cookie -u https://cms-pdmv-dev.cern.ch/mcm/ -o ~/private/dev-cookie.txt --krb --reprocess` +`cern-get-sso-cookie -u https://cms-pdmv-dev.cern.ch/mcm/ -o ~/private/dev-cookie.txt --krb` Run these commands before setting up a CMSSW environment. If these to not work, try to run [getCookie.sh](getCookie.sh). @@ -55,7 +66,7 @@ Usage of manageRequests.py ### Setup PWG -The default PWG in manageRequests.py is set to XXX. To modify your default PWG, change the variable `defaultPWG` on line 23. Alternatively, you can include the flag `-p your_PWG`. +The default PWG in manageRequests.py is set by `mcmscripts_config.py`. To set it to your group, change the configuration file. Alternatively, you can include the flag `-p your_PWG`. ### Creating new requests @@ -105,6 +116,7 @@ Information for requests is provided in a CSV file. The script reads the first l * Gridpack location * Gridpack cards URL * Notes +* McM tag It will also recognize some alternative names. If there is a field title that the script does not recognize, it will complain. The script will ignore columns with the headers "JobId", "Local gridpack location", and "Local LHE" as they are used to supply information to other scripts but do not contain information for McM. @@ -135,6 +147,10 @@ The script can sort for only requests with time per event and size per event set To obtain a list of chained requests from wmLHE requests you can use the `-c` flag. +NEW: a new funcionality has been added to dump much more information (with COLORS!), to be launched with the flag '--listattr'. The default is `-f 0` (Dataset name, Extension, Number of Completed/Total events). The level of verbosity can be increased to `-f 1` (Status, Time Event, CMSSW Release, Priority), `-f 2` (Cross Section, Filter efficiency, Matching efficiency, Tags, Generators, Name of Fragment, Notes), `-f 3` (Last Updater Name and Email, McM View and Edit Links), `-f 4` (Member of the chains including prepIds of the chained requests and direct McM chain link), and `-f 5` (Fragment code). Example: + +`python getRequests.py -listattr 5 "actor=perrozzi&member_of_campaign=*GS*&status=new"` + Usage of testRequests.py ------------------------ diff --git a/addTags.py b/addTags.py new file mode 100755 index 0000000..06acb8a --- /dev/null +++ b/addTags.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python + +################################ +# +# addTags.py +# +# Script to add tags to old requests. +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sys +import os.path +import argparse +import time +import mcmscripts_config +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * # Load class to access McM + + +def getArguments(): + parser = argparse.ArgumentParser( + description='Tag old requests.') + + # Command line flags + parser.add_argument('ids', metavar='PrepIDs', + help='List of PrepIDs to tag copies of.') + parser.add_argument('tag', metavar='Tag') + + args_ = parser.parse_args() + return args_ + + +def fillIDRange(pwg, campaign, first, last): + first = int(first) + last = int(last) + chains = [] + if first > last: + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) + sys.exit(1) + + for number in range(first, last+1): + chains.append("{0}-{1}-{2:05d}".format(pwg, campaign, number)) + return chains + + +def parseIDList(compactList): + splitList = compactList.split(',') + chains = [] + for subList in splitList: + splitSubList = subList.split('-') + if len(splitSubList) == 3: + chains.append(subList) + elif len(splitSubList) == 4: + chains += fillIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[3]) + elif len(splitSubList) == 6: + if splitSubList[0] != splitSubList[3]: + print "Error: PrepID range must be for the same PWG." + sys.exit(1) + if splitSubList[1] != splitSubList[4]: + print "Error: PrepID range must be for the same chained campaign." + sys.exit(1) + chains += fillIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[5]) + else: + print "Error: Poorly formed PrepID list." + sys.exit(1) + return chains + + +def modifyRequest(mcm, req, tag): + for existing in req['tags']: + if existing.startswith("EXO"): + print " \033[0;33m{0} has already been tagged with {1}\033[0;m".format(req['prepid'], existing) + return + req['tags'] += [tag] + answer = mcm.updateA('requests', req) # Update request + if answer['results']: + print " \033[0;32m{0} tagged {1}\033[0;m".format(req['prepid'], tag) + else: + print " \033[0;32m{0} not tagged\033[0;m".format(req['prepid']) + return + + +def tagRequests(prepids, tag): + mcm = restful(dev=False) + campaigns = ['RunIISummer15GS', 'RunIIFall15DR76', 'RunIIFall15MiniAODv1', + 'RunIIFall15MiniAODv2'] + + num = len(prepids) + counter = 0 + + for prepid in prepids: + counter += 1 + time.sleep(1.0) + req = mcm.getA('requests', prepid) + print "{0}/{1} {2} {3}".format(counter, num, prepid, req['dataset_name']) + if req['member_of_campaign'] == "RunIIWinter15wmLHE"\ + or req['member_of_campaign'] == "RunIIWinter15pLHE": + modifyRequest(mcm, req, tag) + dataset_name = req['dataset_name'] + for campaign in campaigns: + query_string = "dataset_name={0}&member_of_campaign={1}".format( + dataset_name, campaign) + failed_to_get = True + for tries in range(3): + time.sleep(0.5) + req_list = mcm.getA('requests', query=query_string) + if req_list is not None: + failed_to_get = False + break + if failed_to_get: + print " \033[0;31mCould not find {0} in {1}\033[0;m".format( + dataset_name, campaign) + continue + if len(req_list) > 1: + print " \033[0;31m{0} has too many requests in {1}\033[0;m".format( + dataset_name, campaign) + continue + if len(req_list) == 0: + print " \033[0;31m{0} does not exist in {1}\033[0;m".format( + dataset_name, campaign) + continue + req = req_list[0] + modifyRequest(mcm, req, tag) + return + + +def main(): + args = getArguments() + prepids = parseIDList(args.ids) + print "Tagging {0} base requests".format(len(prepids)) + tagRequests(prepids, args.tag) + + +if __name__ == '__main__': + main() diff --git a/batchCloneRequests.py b/batchCloneRequests.py new file mode 100755 index 0000000..2741a2c --- /dev/null +++ b/batchCloneRequests.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +################################ +# +# batchCloneRequests.py +# +# Script to clone multiple McM requests. +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sys +import os.path +import argparse +import time +import mcmscripts_config +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * # Load class to access McM + + +def getArguments(): + parser = argparse.ArgumentParser( + description='Clone multiple McM requests.') + + # Command line flags + parser.add_argument('-i', '--ids', dest='ids', help= + 'List of PrepIDs to be cloned.') + parser.add_argument('-c', '--campaign', action='store', dest='campaign', + metavar='name', help='Set member_of_campaign.') + + args_ = parser.parse_args() + return args_ + + +def fillIDRange(pwg, campaign, first, last): + first = int(first) + last = int(last) + requests = [] + if first > last: + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) + sys.exit(1) + + for number in range(first, last+1): + requests.append("{0}-{1}-{2:05d}".format(pwg, campaign, number)) + return requests + + +def parseIDList(compactList): + print compactList + splitList = compactList.split(',') + requests = [] + for subList in splitList: + splitSubList = subList.split('-') + if len(splitSubList) == 3: + requests.append(subList) + elif len(splitSubList) == 4: + requests += fillIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[3]) + elif len(splitSubList) == 6: + if splitSubList[0] != splitSubList[3]: + print "Error: PrepID range must be for the same PWG." + sys.exit(1) + if splitSubList[1] != splitSubList[4]: + print "Error: PrepID range must be for the same chained campaign." + sys.exit(1) + requests += fillIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[5]) + else: + print "Error: Poorly formed PrepID list." + sys.exit(1) + return requests + + +def cloneRequests(requests, campaign): + # Create new requests be cloning an old one based on PrepId + useDev = False + mcm = restful(dev=useDev) # Get McM connection + + print "Adding {0} requests to McM using clone.".format(len(requests)) + for PrepID in requests: + clone_req = mcm.getA('requests', PrepID) # Get request to clone + clone_req['member_of_campaign'] = campaign + answer = mcm.clone(PrepID, clone_req) # Clone request + if answer['results']: + print "\033[0;32m{0} cloned from {1}\033[0;m".format( + answer['prepid'], PrepID) + else: + print "\033[0;31m{0} failed to be cloned to {1}\033[0;m".format( + PrepID, campaign) + time.sleep(0.5) + + +def main(): + args = getArguments() # Setup flags and get arguments + + requests = parseIDList(args.ids) + cloneRequests(requests, args.campaign) + + +if __name__ == '__main__': + main() diff --git a/checkRequests.py b/checkRequests.py index d415792..9670a9e 100755 --- a/checkRequests.py +++ b/checkRequests.py @@ -36,9 +36,8 @@ def fillIDRange(pwg, campaign, first, last): last = int(last) requests = [] if first > last: - print "Error: PrepID range out of order. %s-%s-%05d > %s-%s-%05d" % ( - pwg, campaign, first, pwg, campaign, last) - print "Exiting with status 4." + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) sys.exit(4) for number in range(first, last+1): @@ -59,17 +58,14 @@ def parseIDList(compactList): elif len(splitSubList) == 6: if splitSubList[0] != splitSubList[3]: print "Error: PrepID range must be for the same PWG." - print "Exiting with status 4" sys.exit(4) if splitSubList[1] != splitSubList[4]: print "Error: PrepID range must be for the same campaign." - print "Exiting with status 4" sys.exit(4) requests = requests + fillIDRange(splitSubList[0], splitSubList[1], splitSubList[2], splitSubList[5]) else: print "Error: Poorly formed PrepID list." - print "Exiting with status 3." sys.exit(3) return requests diff --git a/database/README.md b/database/README.md new file mode 100644 index 0000000..3f962d6 --- /dev/null +++ b/database/README.md @@ -0,0 +1,82 @@ +Database +======== + +This is a database to track sets of requests in McM. + +[View database model](database_model.svg) + +Schema +------ + +``` +CREATE TABLE RequestSets( + SetID INTEGER PRIMARY KEY, + Process TEXT, + Tag TEXT, + Events INTEGER, + RequestMultiplicity INTEGER, + Notes TEXT, + Spreadsheet TEXT, + Ticket TEXT); +CREATE TABLE Contacts( + ContactID INTEGER PRIMARY KEY, + Name TEXT NOT NULL, + Email TEXT, + DisplayName TEXT NOT NULL, + UserName TEXT); +CREATE TABLE Requesters( + RequesterID INTEGER PRIMARY KEY, + Name TEXT NOT NULL, + Email TEXT); +CREATE TABLE Instances( + InstanceID INTEGER PRIMARY KEY, + SetID INTEGER, + CampaignChainID INTEGER, + ContactID INTEGER, + RequesterID INTEGER, + PriorityBlock INTEGER, + FOREIGN KEY(SetID) REFERENCES RequestSets(SetID), + FOREIGN KEY(CampaignChainID) REFERENCES CampaignChains(CampaignChainID), + FOREIGN KEY(ContactID) REFERENCES Contacts(ContactID), + FOREIGN KEY(RequesterID) REFERENCES Requesters(RequesterID)); +CREATE TABLE Requests( + RequestsID INTEGER PRIMARY KEY, + CampaignID INTEGER NOT NULL, + New INTEGER NOT NULL, + Validating INTEGER NOT NULL, + Validated INTEGER NOT NULL, + Defined INTEGER NOT NULL, + Approved INTEGER NOT NULL, + Submitted INTEGER NOT NULL, + FOREIGN KEY(CampaignID) REFERENCES Campaigns(CampaignID)); +CREATE TABLE Instance_Requests( + InstanceID INTEGER NOT NULL, + RequestsID INTEGER NOT NULL, + PRIMARY KEY(InstanceID, RequestsID) + FOREIGN KEY(InstanceID) REFERENCES Instances(InstanceID), + FOREIGN KEY(RequestsID) REFERENCES Requests(RequestsID)); +CREATE TABLE SuperCampaigns( + SuperCampaignID INTEGER PRIMARY KEY, + Name TEXT NOT NULL, + Active INTEGER NOT NULL); +CREATE TABLE CampaignChains( + CampaignChainID INTEGER PRIMARY KEY, + Name TEXT, + SuperCampaignID INTEGER NOT NULL, + FOREIGN KEY(SuperCampaignID) REFERENCES SuperCampaigns(SuperCampaignID)); +CREATE TABLE Campaigns( + CampaignID INTEGER PRIMARY KEY, + Name TEXT NOT NULL, + Level INTEGER NOT NULL); +CREATE TABLE CampaignChain_Campaign( + CampaignChainID INTEGER NOT NULL, + CampaignID INTEGER NOT NULL, + OrderInChain INTEGER NOT NULL, + PRIMARY KEY(CampaignChainID, CampaignID), + FOREIGN KEY(CampaignChainID) REFERENCES CampaignChains(CampaignChainID), + FOREIGN KEY(CampaignID) REFERENCES Campaigns(CampaignID)); +CREATE TABLE Settings( + SettingID INTEGER PRIMARY KEY, + Value TEXT, + Description TEXT); +``` diff --git a/database/addContact.py b/database/addContact.py new file mode 100755 index 0000000..ae05e88 --- /dev/null +++ b/database/addContact.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +################################ +# +# addContact.py +# +# Script to add contact to database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import sys +sys.path.append('../') +import mcmscripts_config + + +def interrogate(): + loop = True + while loop: + name = raw_input("Name: ") + email = raw_input("email: ") + display_name = raw_input("Display name: ") + user_name = raw_input("Username: ") + + yn = raw_input("Are these acceptable? [y/n] ") + while True: + if yn.lower() in ["y", "yes"]: + loop = False + break + elif yn.lower() in ["n", "no"]: + break + else: + yn = raw_input('Please enter "yes" or "no". ') + + return [name, email, display_name, user_name] + + +def addContact(answers): + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute('INSERT INTO Contacts VALUES (Null, "{0}", "{1}", "{2}", "{3}");'.format( + answers[0], answers[1], answers[2], answers[3])) + conn.commit() + conn.close() + + print "Sucessfully added {0} to Contacts table.".format(answers[0]) + return + + +def main(): + answers = interrogate() + addContact(answers) + + return + + +if __name__ == '__main__': + main() diff --git a/database/addRequests.py b/database/addRequests.py new file mode 100755 index 0000000..787761d --- /dev/null +++ b/database/addRequests.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python + +################################ +# +# addRequests.py +# +# Script to add requests to database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +import sys +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * +sys.path.append('../') +import mcmscripts_config + + +def Interrogate(): + loop = True + while loop: + process = raw_input("Process: ") + requester = raw_input("Requester name: ") + email = raw_input("Requester email: ") + tag = raw_input("Tag: ") + events = raw_input ("Events: ") + multiplicity = raw_input ("Number of requests: ") + note = raw_input("Notes: ") + spreadsheet = raw_input("Spreadsheet: ") + ticket = raw_input("Ticket: ") + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT SuperCampaignID, Name FROM SuperCampaigns +WHERE Active > 0;""") + out = c.fetchall() + for row in out: + print "{0} {1}".format(row[0], row[1]) + super_campaign = raw_input("SuperCampaign: ") + c.execute("""SELECT CampaignChainID, Name FROM CampaignChains +WHERE SuperCampaignID = {0};""".format(super_campaign)) + out = c.fetchall() + for row in out: + print "{0} {1}".format(row[0], row[1]) + campaign_chain = raw_input("Chain: ") + conn.close() + contact_id = raw_input("Contact: ") + + yn = raw_input("Are these acceptable? [y/n] ") + while True: + if yn.lower() in ["y", "yes"]: + loop = False + break + elif yn.lower() in ["n", "no"]: + break + else: + yn = raw_input('Please enter "yes" or "no". ') + + return [process, requester, email, tag, events, multiplicity, note, + spreadsheet, ticket, super_campaign, campaign_chain, contact_id] + + +def AddRequester(name_, email_): + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute('INSERT INTO Requesters (Name, Email) VALUES ("{0}", "{1}");'.format( + name_, email_)) + requester_id = c.lastrowid + conn.commit() + conn.close() + + print "Added new requester: {0} ({1})".format(name_, email_) + + return requester_id + + +def GetRequester(name_, email_): + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT RequesterID, Name, Email FROM Requesters +WHERE Name = "{0}" OR Email = "{1}";""".format(name_, email_)) + out = c.fetchall() + conn.close() + requester_id = 0 + if len(out) == 0: + requester_id = AddRequester(name_, email_) + elif len(out) == 1: + requester_id = out[0][0] + else: + print "Found more than one requester. Choose the ID to use. 0 creates new requester." + for row in out: + print "{0} {1} {2}".format(row[0], row[1], row[2]) + requester_id = raw_input("User ID: ") + if requester_id == 0: + requester_id = AddRequester(name_, email_) + return requester_id + + +def AddRequestSet(answers_, requester_id_): + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""INSERT INTO RequestSets (Process, Tag, Events, +RequestMultiplicity, Notes, Spreadsheet, Ticket) VALUES ("{0}", "{1}", {2}, {3}, +"{4}", "{5}", "{6}");""".format(answers_[0], answers_[3], answers_[4], answers_[5], + answers_[6], answers_[7], answers_[8])) + set_id = c.lastrowid + c.execute("""INSERT INTO Instances (SetID, CampaignChainID, ContactID, +RequesterID, PriorityBlock) VALUES ({0}, {1}, {2}, {3}, {4});""".format( + set_id, answers_[10], answers_[11], requester_id_, 3)) + instance_id = c.lastrowid + c.execute("""SELECT Campaigns.CampaignID FROM CampaignChains +INNER JOIN CampaignChain_Campaign USING(CampaignChainID) +INNER JOIN Campaigns USING(CampaignID) WHERE CampaignChainID = {0};""".format(answers_[10])) + out = c.fetchall() + for campaign in out: + c.execute("""INSERT INTO Requests (CampaignID, New, Validating, +Validated, Defined, Approved, Submitted) VALUES ({0}, 0, 0, 0, 0, 0, 0);""".format(campaign[0])) + requests_id = c.lastrowid + c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) +VALUES ({0}, {1});""".format(instance_id, requests_id)) + conn.commit() + conn.close() + + print "Sucessfully added {0} {1} to RequestSets table.".\ + format(answers_[0], answers_[1]) + return + + +def main(): + answers = Interrogate() + requester_id = GetRequester(answers[1], answers[2]) + AddRequestSet(answers, requester_id) + + return + + +if __name__ == '__main__': + main() diff --git a/database/batchAddRequests.py b/database/batchAddRequests.py new file mode 100755 index 0000000..025a3f8 --- /dev/null +++ b/database/batchAddRequests.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +################################ +# +# batchAddRequests.py +# +# Script to add requests to database with a CSV file +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +import sys +import csv +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * +import time +sys.path.append('../') +import mcmscripts_config + + +def addRequestSet(answers, mcm): + req_list = mcm.getA('requests', query='tags={0}&member_of_campaign=RunIISummer15GS'.format( + answers[1])) + multiplicity = len(req_list) + events = 0 + for req in req_list: + events += req['total_events'] + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute('INSERT INTO RequestSets (Process, Tag, Events, Notes, Spreadsheet, RequestMultiplicity) VALUES ("{0}", "{1}", {2}, "{3}", "{4}", {5});'.format( + answers[0], answers[1], events, answers[6], answers[5], multiplicity)) + conn.commit() + conn.close() + + print "Sucessfully added {0} {1} to RequestSets table.".\ + format(answers[0], answers[1]) + return + +def main(): + csvfile = open('fill.csv', 'r') + mcm = restful(dev=False) + print "Adding:" + for row in csv.reader(csvfile): + if row[0].startswith("#"): + continue + print row[0] + addRequestSet(row, mcm) + time.sleep(1) + + return + + +if __name__ == '__main__': + main() diff --git a/database/check.py b/database/check.py new file mode 100755 index 0000000..6d20f6b --- /dev/null +++ b/database/check.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python + +################################ +# +# check.py +# +# Script to check status of requests and save in database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +import sys +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * +import time +sys.path.append('../') +import mcmscripts_config + + +def getArguments(): + parser = argparse.ArgumentParser( + description='Check status of requests in McM and save in database.') + + # Command line flags + parser.add_argument('--check', action='store', dest='check', + help='Type of check to perform. 0 Check all sets, 1 Check sets that are not finished, 2 Check requests being validated.') + parser.add_argument('-t', '--tag', action='store', dest='tag', + help='Check status of requests based on tag.') + + args_ = parser.parse_args() + return args_ + + +def getRequestSets(): + mcm = restful(dev=False) + + wmLHE_campaign = "RunIIWinter15wmLHE" + pLHE_campaign = "RunIIWinter15pLHE" + GS_campaign = "RunIISummer15GS" + DR_campaign = "RunIIFall15DR76" + Mini_campaign = "RunIIFall15MiniAODv1" + Miniv2_campaign = "RunIIFall15MiniAODv2" + status_name = [["LHE_New", "LHE_Validating", "LHE_Validated", "LHE_Defined", + "LHE_Approved", "LHE_Submitted", "LHE_Done"], + ["GS_New", "GS_Validating", "GS_Validated", "GS_Defined", + "GS_Approved", "GS_Submitted", "GS_Done"], + ["DR_New", "DR_Validating", "DR_Validated", "DR_Defined", + "DR_Approved", "DR_Submitted", "DR_Done"], + ["MiniAOD_New", "MiniAOD_Validating", "MiniAOD_Validated", + "MiniAOD_Defined", "MiniAOD_Approved", "MiniAOD_Submitted", + "MiniAOD_Done"], + ["MiniAODv2_New", "MiniAODv2_Validating", + "MiniAODv2_Validated", "MiniAODv2_Defined", + "MiniAODv2_Approved", "MiniAODv2_Submitted", + "MiniAODv2_Done"]] + + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute('SELECT SetID, Tag FROM RequestSets WHERE RequestMultiplicity != MiniAODv2_Done') + out = c.fetchall() + + print "Checking:" + for request in out: + print request[1] + + campaigns = [wmLHE_campaign, GS_campaign, DR_campaign, Mini_campaign, Miniv2_campaign] + req_list = mcm.getA('requests', query='tags={0}'.format( + request[1])) + statuses = [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0]] + for req in req_list: + for i in range(len(campaigns)): + if req['member_of_campaign'] == campaigns[i]: + if req['approval'] == "none" and req['status'] == "new": + statuses[i][0] += 1 + elif req['approval'] == "validation" and req['status'] == "new": + statuses[i][1] += 1 + elif req['approval'] == "validation" and req['status'] == "validation": + statuses[i][2] += 1 + elif req['approval'] == "define" and req['status'] == "defined": + statuses[i][3] += 1 + elif req['approval'] == "approve" and req['status'] == "approved": + statuses[i][4] += 1 + elif req['approval'] == "submit" and req['status'] == "submitted": + statuses[i][5] += 1 + elif req['approval'] == "submit" and req['status'] == "done": + statuses[i][6] += 1 + #print statuses + #print request[0] + for i in range(len(statuses)): + for j in range(len(statuses[0])): + c.execute('UPDATE RequestSets SET {0} = {1} WHERE SetID = {2}'.format( + status_name[i][j], statuses[i][j], request[0])) + time.sleep(1) + conn.commit() + conn.close() + + return + + +def checkRequests(): + mcm = restful(dev=False) + + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT DISTINCT Tag, RequestMultiplicity, Done, Campaigns.Name, +RequestsID FROM Requests INNER JOIN Campaigns USING(CampaignID) +INNER JOIN Instance_Requests USING(RequestsID) INNER JOIN Instances +USING(InstanceID) INNER JOIN RequestSets USING(SetID);""") + out = c.fetchall() + + status_names = ["New", "Validating", "Validated", "Defined", "Approved", + "Submitted", "Done"] + + for row in out: + if row[1] == row[2]: + continue + print "{0} {1}".format(row[0], row[3]) + req_list = mcm.getA('requests', + query='tags={0}&member_of_campaign={1}'.format( + row[0], row[3])) + time.sleep(1) + statuses = [0, 0, 0, 0, 0, 0, 0] + for req in req_list: + if req['approval'] == "none" and req['status'] == "new": + statuses[0] += 1 + elif req['approval'] == "validation" and req['status'] == "new": + statuses[1] += 1 + elif req['approval'] == "validation" and req['status'] == "validation": + statuses[2] += 1 + elif req['approval'] == "define" and req['status'] == "defined": + statuses[3] += 1 + elif req['approval'] == "approve" and req['status'] == "approved": + statuses[4] += 1 + elif req['approval'] == "submit" and req['status'] == "submitted": + statuses[5] += 1 + elif req['approval'] == "submit" and req['status'] == "done": + statuses[6] += 1 + + for i, status_name in enumerate(status_names): + c.execute("""UPDATE Requests SET {0} = {1} WHERE RequestsID = {2}""".format( + status_name, statuses[i], row[4])) + + c.execute("""\ +UPDATE Settings +SET Value = "{0}" +WHERE SettingID = 1;""".format(time.asctime())) + + conn.commit() + conn.close() + + return + + +def main(): + args = getArguments() # Setup flags and get arguments + #getRequestSets() + checkRequests() + + return + + +if __name__ == '__main__': + main() diff --git a/database/database_model.svg b/database/database_model.svg new file mode 100644 index 0000000..7b6e014 --- /dev/null +++ b/database/database_model.svg @@ -0,0 +1,1477 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/database/favicon.ico b/database/favicon.ico new file mode 100644 index 0000000..4b21e0a Binary files /dev/null and b/database/favicon.ico differ diff --git a/database/global.css b/database/global.css new file mode 100644 index 0000000..47574f0 --- /dev/null +++ b/database/global.css @@ -0,0 +1,231 @@ +/************************* +** +** Global CSS for Exotica MC Status +** +** +*************************/ + +/**************** +** +** Page +** +****************/ + +body { + background-color: #fbfbfb; + padding: 0; + margin: 0; + border: 0; + font-family: Helvetica, Ariel, sans-serif; + font-size: 12pt; +} + +div.wrapper { + margin: 20px 50px 20px 50px; +} + +/**************** +** +** Text +** +****************/ + +a, a:visited { + text-decoration: none; + color: #0000dd; +} + +a:hover, a:visited:hover { + color: #dd0000; +} + +h1 { + font-size: 32pt; + color: #000000; +} + +h2 { + font-size: 24pt; + background-color: #84a4aa; + color: #000000; + padding: 6pt; + margin-top: 24pt; +} + +div.toc h2 { + font-size: 18pt; + background-color: #fbfbfb; + color: #000000; + padding: 0px; + margin-bottom: 10pt; + margin-top: 0px; +} + +li { + margin: 10pt 0pt 10pt 0pt; +} + +div.update-time { + color: #999999; + font-size: 10pt; + font-style: italic; + margin: 15pt 0 15pt 0; +} + +/**************** +** +** Table +** +****************/ + +table { + border-collapse: collapse; +} + +tbody tr:nth-child(odd) { + background: #ffffff; +} + +tbody tr:nth-child(even) { + background: #dddddd; +} + +thead { + border-bottom: 2px solid black; +} + +thead tr { + background-color: #c0bcb9; + color: #000000; +} + +th { + padding: 8pt 2pt 6pt 2pt; +} + +td { + padding-top: 8px; + padding-bottom: 8px; +} + +th.lhe, th.gs, th.dr, th.miniaod, th.miniaodv2 { + min-width: 100px; +} + +th.notes { + padding-right: 5pt; +} + +td.lhe, td.gs, td.dr, td.miniaod, td.miniaodv2 { + text-align: center; + padding-left: 2px; + padding-right: 2px; +} + +td.process, td.tag, td.requester, td.contact, td.events { + padding-left: 5pt; + padding-right: 5pt; +} + +/**************** +** +** Bar charts +** +****************/ + +svg.status { + margin-top: -18px; +} + +rect.new, path.new { + fill: #ccccff; + stroke-width: 0; +} + +rect.validating, path.validating { + fill: #cc99ff; + stroke-width: 0; +} + +rect.validated, path.validated { + fill: #6ba6e8; + stroke-width: 0; +} + +rect.defined, path.defined { + fill: #52fbc4; + stroke-width: 0; +} + +rect.approved, path.approved { + fill: #ffeba4; + stroke-width: 0; +} + +rect.submitted, path.submitted { + fill: #ffc570; + stroke-width: 0; +} + +rect.done, path.done { + fill: #66aa66; + stroke-width: 0; +} + +rect.absent, path.absent { + fill: #727272; + stroke-width: 0; +} + +rect.extra, path.extra { + fill: #cc0000; + stroke-width: 0; +} + +g.tooltip text { + font-size: 13px; +} + +/**************** +** +** Sprites +** +****************/ + +span.spreadsheet_icon { + display: block; + width: 30px; + height: 30px; + background: url(sprites.png) -6px -6px; + margin: auto; +} + +span.spreadsheet_icon:hover { + background: url(sprites.png) -48px -6px; +} + +span.note_icon { + display:block; + width: 30px; + height: 30px; + background: url(sprites.png) -6px -48px; + margin: auto; +} + +span.note_icon:hover { + background: url(sprites.png) -48px -48px; +} + +span.note_content { + display: none; + background-color: #f2f2ee; + border: 1px solid; + border-color: #000000; + position: absolute; + padding: 7px; + -moz-transform: translateX(-100%) translateY(-15px); + -ms-transform: translateX(-100%) translateY(-15px); + -webkit-transform: translateX(-100%) translateY(-15px); + -o-transform: translateX(-100%) translateY(-15px); + transform: translateX(-100%) translateY(-15px); +} diff --git a/database/makeHTML.py b/database/makeHTML.py new file mode 100755 index 0000000..5d0552a --- /dev/null +++ b/database/makeHTML.py @@ -0,0 +1,511 @@ +#!/usr/bin/env python + +################################ +# +# check.py +# +# Script to check status of requests and save in database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +import sys +import math +import shutil +sys.path.append('../') +import mcmscripts_config + +# Global variables +object_counter = 0 + + +def display_number(n): + if n == 0: + return "0" + else: + prefix = ['', 'k', 'M'] + n = float(n) + prefix_id = max(0, min(2, int(math.floor(math.log10(abs(n))/3.0)))) + return "{0:.5g}{1}".format(n/10**(3*prefix_id), prefix[prefix_id]); + + +def writeRequests(page, fout, c, super_campaign, request_set, instance): + campaign_classes = ["lhe", "gs", "dr", "miniaod"] + global object_counter + + c.execute("""\ +SELECT Campaigns.Name, + New, + Validating, + Validated, + Defined, + Approved, + Submitted, + Done +FROM Requests +INNER JOIN Instance_Requests USING(RequestsID) +INNER JOIN Campaigns USING(CampaignID) +WHERE InstanceID = {0} +ORDER BY Level""".format(instance[0])) + requests = c.fetchall() + class_offset = 0 + if len(requests) < 4: + fout.write("  \n") + class_offset = 1 + for i in range(len(requests)): + if page == 0: + fout.write(" \n".format(campaign_classes[i + class_offset])) + fout.write("""\ + + + +""".format(request_set[2], requests[i][0])) + statuses = ["new", "validating", "validated", "defined", "approved", + "submitted", "done", "absent", "extra"] + sum_known = requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4] + requests[i][5] + requests[i][6] + requests[i][7] + total = max(request_set[4], sum_known) + x_pos = 0.0 + tmp_object_counter = object_counter + for j in range(7): + bar_width = 0.0 + if total != 0: + if requests[i][j+1] <= request_set[4]: + bar_width = float(requests[i][j+1])/float(total)*100.0 + else: + bar_width = float(request_set[4])/float(total)*100.0 + if bar_width < 0.0: + bar_width = 0.0 + fout.write("""\ + +""".format(x_pos, bar_width, statuses[j], tmp_object_counter)) + x_pos += bar_width + tmp_object_counter += 1 + unkown_reason = "absent" + if total > request_set[4]: + unkown_reason = "extra" + # if x_pos >= 95.0: + # x_pos = 95.0 + fout.write("""\ + +""".format(x_pos, max(100.0 - x_pos, 0.0), unkown_reason, tmp_object_counter)) + fout.write("""\ + + +""") + for j in range(7): + fout.write("""\ + + + {2} {1} + +""".format(object_counter, statuses[j], requests[i][j+1])) + object_counter += 1 + fout.write("""\ + + + {2} {1} + +""".format(object_counter, unkown_reason, max(total - sum_known, sum_known - request_set[4]))) + object_counter += 1 + fout.write("""\ + +""") + fout.write(" \n") + elif page == 1: + fout.write(" \n".format(campaign_classes[i + class_offset])) + fout.write("""\ + + + +""".format(request_set[2], requests[i][0])) + statuses = ["new", "validating", "validated", "defined", "approved", + "submitted", "done", "absent", "extra"] + sum_known = requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4] + requests[i][5] + requests[i][6] + requests[i][7] + total = max(request_set[4], sum_known) + combined_statuses = ["preparation", "ready to run", "running", + "done", "absent", "extra"] + combined_classes = [statuses[0], statuses[4], statuses[5], statuses[6]] + combined_requests = [requests[i][1] + requests[i][2] + requests[i][3], + requests[i][4] + requests[i][5], requests[i][6], + requests[i][7]] + x_pos = 0.0 + tmp_object_counter = object_counter + for j in range(4): + bar_width = 0.0 + if total != 0: + bar_width = float(combined_requests[j])/float(total)*100.0 + if bar_width < 0.0: + bar_width = 0.0 + fout.write("""\ + +""".format(x_pos, bar_width, combined_classes[j], tmp_object_counter)) + x_pos += bar_width + tmp_object_counter += 1 + unkown_reason = "absent" + if total > request_set[4]: + unkown_reason = "extra" + fout.write("""\ + +""".format(x_pos, max(100.0 - x_pos, 0.0), unkown_reason, tmp_object_counter)) + fout.write("""\ + + +""") + for j in range(4): + fout.write("""\ + + + {2} {3} + +""".format(object_counter, combined_classes[j], combined_requests[j], + combined_statuses[j])) + object_counter += 1 + fout.write("""\ + + + {2} {1} + +""".format(object_counter, unkown_reason, total - sum_known)) + object_counter += 1 + fout.write("""\ + +""") + fout.write(" \n") + return + + +def writeInstances(page, fout, c, super_campaign, request_set): + c.execute("""\ +SELECT InstanceID, + DisplayName, + Requesters.Name, + Requesters.Email, + CampaignChains.Name +FROM Instances +INNER JOIN Contacts USING(ContactID) +INNER JOIN Requesters USING(RequesterID) +INNER JOIN CampaignChains USING(CampaignChainID) +WHERE SetID = {0} + AND SuperCampaignID = {1};""".format(request_set[0], super_campaign[0])) + instances = c.fetchall() + for instance in instances: + fout.write("\n") + if page == 0: + fout.write("""\ + {0} + {1} + {3} + {4} + {5} + {6} +""".format(request_set[1], request_set[2], instance[3], instance[2], + instance[1], display_number(request_set[3]), request_set[4])) + elif page == 1: + fout.write("""\ + {0} + {2} +""".format(request_set[1], instance[3], instance[2])) + writeRequests(page, fout, c, super_campaign, request_set, instance) + if page == 0: + if request_set[6] == "": + fout.write("  \n") + else: + fout.write("  \n".format( + request_set[6])) + if request_set[5] == "": + fout.write("  \n") + else: + fout.write("  {0}\n".format(request_set[5])) + elif page == 1: + if request_set[6] == "": + fout.write("  \n") + else: + fout.write("  \n".format( + request_set[6])) + if request_set[5] == "": + fout.write("  \n") + else: + fout.write("  {0}\n".format(request_set[5])) + fout.write("\n") + return + + +def writeRequestSets(page, fout, c, super_campaign): + c.execute("""\ +SELECT DISTINCT SetID, + Process, + Tag, + Events, + RequestMultiplicity, + Notes, + Spreadsheet +FROM RequestSets +INNER JOIN Instances USING(SetID) +INNER JOIN CampaignChains USING(CampaignChainID) +WHERE SuperCampaignID = {0};""".format(super_campaign[0])) + request_sets = c.fetchall() + for request_set in request_sets: + writeInstances(page, fout, c, super_campaign, request_set) + return + + +def writeSuperCampaigns(page, fout, c): + global object_counter + object_counter = 0 + c.execute("""\ +SELECT SuperCampaignID, + Name +FROM SuperCampaigns +ORDER BY Active;""") + super_campaigns = c.fetchall() + fout.write("""\ +
+

Campaigns:

+ +
+ +""") + + for super_campaign in super_campaigns: + fout.write("""\ +

{1}

+ + + +""".format(super_campaign[1].replace(" ", "-"), super_campaign[1])) + if page == 0: + fout.write("""\ + + + + + + + + + + + + +""") + elif page == 1: + fout.write("""\ + + + + + + + + +""") + fout.write("\n\n\n") + + writeRequestSets(page, fout, c, super_campaign) + + fout.write("\n
ProcessTagRequesterContactEventsRequestsLHEGSDRMiniAODSpreadsheetNotesProcessRequesterLHEGSDRMiniAODSpreadsheetNotes
\n") + + return + + +def makeAnalyzerHTML(): + fout = open('{0}analyzer.html'.format(mcmscripts_config.html_location), 'w') + campaign_classes = ["lhe", "gs", "dr", "miniaod", "miniaodv2"] + + fout.write("""\ + + + + + Exotica MC Status for Analyzers + + + + + + + + + + + + + + + + +
+

Exotica MC

+ +""") + + conn = sqlite3.connect(mcmscripts_config.database_location) + + c = conn.cursor() + writeSuperCampaigns(1, fout, c) + + c.execute("""\ +SELECT Value +FROM Settings +WHERE SettingID = 1""") + check_time = c.fetchone() + fout.write("""\ + +
Updated {0}

+
+ + + +""".format(check_time[0])) + fout.close() + + print "Generated analyzer page" + + return + + +def makeContactHTML(): + fout = open('{0}contact.html'.format(mcmscripts_config.html_location), 'w') + campaign_classes = ["lhe", "gs", "dr", "miniaod", "miniaodv2"] + + fout.write("""\ + + + + + Exotica MC Status for Contacts + + + + + + + + + + + + + + + + +
+

Exotica MC

+ +

Go to analyzer page

+ +""") + + # + # + + conn = sqlite3.connect(mcmscripts_config.database_location) + + c = conn.cursor() + writeSuperCampaigns(0, fout, c) + + c.execute("""\ +SELECT Value +FROM Settings +WHERE SettingID = 1""") + check_time = c.fetchone() + fout.write("""\ + +
Updated {0}

+
+ + + +""".format(check_time[0])) + fout.close() + + print "Generated contact page" + + return + + +def syncAuxiliaryFiles(): + shutil.copyfile("global.css", "{0}global.css".format(mcmscripts_config.html_location)) + shutil.copyfile("favicon.ico", "{0}favicon.ico".format(mcmscripts_config.html_location)) + shutil.copyfile("sprites.png", "{0}sprites.png".format(mcmscripts_config.html_location)) + + print "Copied auxiliary files" + + return + + +def main(): + makeAnalyzerHTML() + makeContactHTML() + syncAuxiliaryFiles() + + return + + +if __name__ == '__main__': + main() diff --git a/database/newInstance.py b/database/newInstance.py new file mode 100755 index 0000000..ded3cd0 --- /dev/null +++ b/database/newInstance.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python + +################################ +# +# newInstance.py +# +# Script to add new instance of set of requests to database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +import sys +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * +sys.path.append('../') +import mcmscripts_config + + +# def AddRequester(name_, email_): +# conn = sqlite3.connect(mcmscripts_config.database_location) +# c = conn.cursor() +# c.execute('INSERT INTO Requesters (Name, Email) VALUES ("{0}", "{1}");'.format( +# name_, email_)) +# requester_id = c.lastrowid +# conn.commit() +# conn.close() + +# print "Added new requester: {0} ({1})".format(name_, email_) + +# return requester_id + + +# def GetRequester(name_, email_): +# conn = sqlite3.connect(mcmscripts_config.database_location) +# c = conn.cursor() +# c.execute("""SELECT RequesterID, Name, Email FROM Requesters +# WHERE Name = "{0}" OR Email = "{1}";""".format(name_, email_)) +# out = c.fetchall() +# conn.close() +# requester_id = 0 +# if len(out) == 0: +# requester_id = AddRequester(name_, email_) +# elif len(out) == 1: +# requester_id = out[0][0] +# else: +# print "Found more than one requester. Choose the ID to use. 0 creates new requester." +# for row in out: +# print "{0} {1} {2}".format(row[0], row[1], row[2]) +# requester_id = raw_input("User ID: ") +# if requester_id == 0: +# requester_id = AddRequester(name_, email_) +# return requester_id + + +# def AddRequestSet(answers_, requester_id_): +# conn = sqlite3.connect(mcmscripts_config.database_location) +# c = conn.cursor() +# c.execute("""INSERT INTO RequestSets (Process, Tag, Events, +# RequestMultiplicity, Notes, Spreadsheet, Ticket) VALUES ("{0}", "{1}", {2}, {3}, +# "{4}", "{5}", "{6}");""".format(answers_[0], answers_[3], answers_[4], answers_[5], +# answers_[6], answers_[7], answers_[8])) +# set_id = c.lastrowid +# c.execute("""INSERT INTO Instances (SetID, CampaignChainID, ContactID, +# RequesterID, PriorityBlock) VALUES ({0}, {1}, {2}, {3}, {4});""".format( +# set_id, answers_[10], answers_[11], requester_id_, 3)) +# instance_id = c.lastrowid +# c.execute("""SELECT Campaigns.CampaignID FROM CampaignChains +# INNER JOIN CampaignChain_Campaign USING(CampaignChainID) +# INNER JOIN Campaigns USING(CampaignID) WHERE CampaignChainID = {0};""".format(answers_[10])) +# out = c.fetchall() +# for campaign in out: +# c.execute("""INSERT INTO Requests (CampaignID, New, Validating, +# Validated, Defined, Approved, Submitted) VALUES ({0}, 0, 0, 0, 0, 0, 0);""".format(campaign[0])) +# requests_id = c.lastrowid +# c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) +# VALUES ({0}, {1});""".format(instance_id, requests_id)) +# conn.commit() +# conn.close() + +# print "Sucessfully added {0} {1} to RequestSets table.".\ +# format(answers_[0], answers_[1]) +# return + + +def main(): + loop = True + while loop: + tag = raw_input("Tag: ") + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT SetID, Process, Events, RequestMultiplicity, Notes, +Spreadsheet, Ticket, ContactID, DisplayName, RequesterID, Requesters.Name, +PriorityBlock, CampaignChains.Name, SuperCampaigns.Name, InstanceID, CampaignChainID +FROM RequestSets INNER JOIN Instances USING(SetID) INNER JOIN CampaignChains +USING(CampaignChainID) INNER JOIN SuperCampaigns USING(SuperCampaignID) +INNER JOIN Contacts USING(ContactID) INNER JOIN Requesters USING(RequesterID) +WHERE Tag = "{0}";""".format( + tag)) + out = c.fetchall() + instance_list = [row[14] for row in out] + print instance_list + print "Process: {0}".format(out[0][1]) + print "Events: {0}".format(out[0][2]) + print "Request Multiplicity: {0}".format(out[0][3]) + if out[0][4] != "": + print "Notes: {0}".format(out[0][4]) + print "Spreadsheet: {0}".format(out[0][5]) + print "Ticket: {0}".format(out[0][6]) + for i, row in enumerate(out): + print "{0} ----------".format(i) #row[14]) + print "Contact: {1}".format(row[7], row[8]) + print "Requester: {1}".format(row[9], row[10]) + print "Block: ".format(row[11]) + print "SuperCampaign: {0}".format(row[13]) + print "Chain: {0}".format(row[12]) + print "------------" + conn.close() + + print "" + row_num = int(raw_input("Instance to copy: ")) + chainID = int(raw_input("Chain ID (7,11 GS / 9,13 wmLHE / 10,14 pLHE): ")) + + yn = raw_input("Are these acceptable? [y/n] ") + while True: + if yn.lower() in ["y", "yes"]: + loop = False + break + elif yn.lower() in ["n", "no"]: + break + else: + yn = raw_input('Please enter "yes" or "no". ') + + print "\n===========\n" + + SetID = out[row_num][0] + ContactID = out[row_num][7] + RequesterID = out[row_num][9] + PriorityBlock = out[row_num][11] + old_InstanceID = out[row_num][14] + + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""INSERT INTO Instances (SetID, CampaignChainID, ContactID, +RequesterID, PriorityBlock) VALUES ({0}, {1}, {2}, {3}, {4});""".format( + SetID, chainID, ContactID, RequesterID, PriorityBlock)) + new_InstanceID = c.lastrowid + print "Added new instance" + + c.execute("""SELECT CampaignID FROM CampaignChain_Campaign +WHERE CampaignChainID = {0};""".format(chainID)) + out = c.fetchall() + for row in out: + c.execute("""SELECT CampaignID, RequestsID FROM Requests +INNER JOIN Instance_Requests USING(RequestsID) WHERE InstanceID = {0} +AND CampaignID = {1};""".format(old_InstanceID, row[0])) + out_requests = c.fetchall() + if (len(out_requests) > 0): + c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) +VALUES ({0}, {1});""".format(new_InstanceID, out_requests[0][1])) + print "Added existing request to instance" + else: + c.execute("""INSERT INTO Requests (CampaignID, New, Validating, +Validated, Defined, Approved, Submitted) VALUES ({0}, 0, 0, 0, 0, 0, 0);""".format( + row[0])) + new_RequestsID = c.lastrowid + c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) +VALUES ({0}, {1});""".format(new_InstanceID, new_RequestsID)) + print "Added new request to instance" + + conn.commit() + conn.close() + + return + + +if __name__ == '__main__': + main() diff --git a/database/sprites.png b/database/sprites.png new file mode 100644 index 0000000..5470b64 Binary files /dev/null and b/database/sprites.png differ diff --git a/getRequests.py b/getRequests.py index 8e4e835..ed34a81 100755 --- a/getRequests.py +++ b/getRequests.py @@ -6,7 +6,8 @@ # # Script to get a list of requests from McM # -# author: David G. Sheffield (Rutgers) +# authors: David G. Sheffield (Rutgers) +# Luca Perrozzi (ETHZ) # ################################ @@ -19,18 +20,65 @@ sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import * # Load class to access McM from requestClass import * # Load class to store request information +import mcmscripts_config + +class bcolorClass: + MAGENTA = '\033[35m' + BLUE = '\033[34m' + GREEN = '\033[32m' + # RED = '\033[31m' + YELLOW = '\033[33m' + WHITE = '\033[1;37m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + Gray_like_Ghost = '\033[1;30m' + RED = '\033[1;31m' + Green_like_Grass = '\033[1;32m' + Yellow_like_Yolk = '\033[1;33m' + Blue_like_Blood = '\033[1;34m' + Magenta_like_Mimosa = '\033[1;35m' + CYAN = '\033[1;36m' + Crimson_like_Chianti = '\033[1;38m' + Highlighted_Red_like_Radish = '\033[1;41m' + Highlighted_Green_like_Grass = '\033[1;42m' + Highlighted_Brown_like_Bear = '\033[1;43m' + Highlighted_Blue_like_Blood = '\033[1;44m' + Highlighted_Magenta_like_Mimosa = '\033[1;45m' + Highlighted_Cyan_like_Caribbean = '\033[1;46m' + Highlighted_Gray_like_Ghost = '\033[1;47m' + Highlighted_Crimson_like_Chianti = '\033[1;48m' + + def _noColor(self): + for m in dir(self): + if not m.startswith('_'): + setattr(self, m, '') + + def __init__(self): + if mcmscripts_config.auto_bw and not sys.stdout.isatty(): + self._noColor() + +bcolors = bcolorClass() def getArguments(): - parser = argparse.ArgumentParser(description='Get a list of PrepIDs from McM based on a query.') + parser = argparse.ArgumentParser( + description='Get a list of PrepIDs from McM based on a query.') # Command line flags parser.add_argument('query') - parser.add_argument('-n', action='store_true', dest='getNew', help='Only get requests with unmodified time and size per event.') - parser.add_argument('-v', action='store_true', dest='getForValidation', help='Only get requests with positive time and size per event.') - parser.add_argument('-c', action='store_true', dest='getChain', help='Return PrepID of chain.') + parser.add_argument('-n', action='store_true', dest='getNew', + help='Only get requests with unmodified time and size per event.') + parser.add_argument('-v', action='store_true', dest='getForValidation', + help='Only get requests with positive time and size per event.') + parser.add_argument('-c', action='store_true', dest='getChain', + help='Return PrepID of chain.') + parser.add_argument('-listattr', dest='listAttr', type=int, default=-1, + help='List attributes for each PrepID. 0 (default) to 5 in increasing level of verbosity') + parser.add_argument('-bw', action='store_true', dest='noColors', default=False, + help='B/W output') parser.add_argument('-f', dest='format', type=int, default=0, - help='Format of output. 0 (default) = input for scripts, 1 = human-readable, 2 = HTML') + help='Format of output. 0 (default) = input for scripts, 1 = human-readable, 2 = HTML, 3 = write every PrepID') args_ = parser.parse_args() return args_ @@ -39,16 +87,103 @@ def getArguments(): def checkFile(file_): # Check that CSV file exists if not os.path.isfile(file_): - print "Error: File %s does not exist." % file_ - print "Exiting with status 1." + print "Error: File {0} does not exist.".format(file_) sys.exit(1) -def getPrepIDList(query_string, getNew, getForValidation, getChain): +def getMcMlist(query_string,printout): useDev = False - mcm = restful( dev=useDev ) # Get McM connection - print query_string + mcm = restful(dev=useDev) # Get McM connection + if printout: + print 'MCM query string: {0}{1}{2}'.format(bcolors.MAGENTA, + query_string, bcolors.ENDC) req_list = mcm.getA('requests', query=query_string) + return req_list + +def getPrepIDListWithAttributes(query_string,listAttr): + req_list = getMcMlist(query_string,True) + print '\n' + print '================================================================================\n' + for req in req_list: + if listAttr > 5: # full dump of the request object, useful for debugging purpose + print bcolors.MAGENTA +\ + 'prepid='+ bcolors.ENDC,req['prepid'],\ + ''+ bcolors.ENDC + pprint.pprint(req) + print '' + else: + print bcolors.MAGENTA +\ + 'prepid='+ bcolors.ENDC,req['prepid'],\ + ', '+bcolors.MAGENTA+'Dataset name='+ bcolors.ENDC,req['dataset_name'],\ + ', '+bcolors.MAGENTA+'Extension='+ bcolors.ENDC,req['extension'],\ + ', '+bcolors.MAGENTA+'Completed/Total events='+ bcolors.ENDC,str(req['completed_events'])+'/'+str(req['total_events']),\ + ''+ bcolors.ENDC + if listAttr > 0: + print bcolors.RED +\ + 'Approval='+ bcolors.ENDC,req['approval'],\ + ', '+bcolors.RED+'Status='+ bcolors.ENDC,req['status'],\ + ', '+bcolors.RED+'Time Event='+ bcolors.ENDC,req['time_event'],\ + ', '+bcolors.RED+'CMSSW Release='+ bcolors.ENDC,req['cmssw_release'],\ + ', '+bcolors.RED+'Priority='+ bcolors.ENDC,req['priority'],\ + ''+ bcolors.ENDC + if listAttr > 1: + if(len(req['generator_parameters'])>0): + print bcolors.GREEN +\ + 'Cross Section='+ bcolors.ENDC,req['generator_parameters'][0]['cross_section'],'pb',\ + ', '+bcolors.GREEN+'Filter efficiency='+ bcolors.ENDC,str(req['generator_parameters'][0]['filter_efficiency'])+' +/- '+str(req['generator_parameters'][0]['filter_efficiency_error']),\ + ', '+bcolors.GREEN+'Match efficiency='+ bcolors.ENDC,str(req['generator_parameters'][0]['match_efficiency'])+' +/- '+str(req['generator_parameters'][0]['match_efficiency_error']),\ + ''+ bcolors.ENDC + else: + print bcolors.GREEN +\ + 'Cross Section= -1 pb',\ + ', Filter efficiency= -1',\ + ', Match efficiency= -1',\ + ''+ bcolors.ENDC + print bcolors.CYAN +\ + 'Tags='+ bcolors.ENDC,str(req['tags']).replace("u'",'').replace("'",""),\ + ', '+bcolors.CYAN+'Generators='+ bcolors.ENDC,req['name_of_fragment'],\ + ', '+bcolors.CYAN+'Name of Fragment='+ bcolors.ENDC,req['name_of_fragment'],\ + ', '+bcolors.CYAN+'Notes='+ bcolors.ENDC,req['notes'],\ + ''+ bcolors.ENDC + if listAttr > 2: + print bcolors.BLUE +\ + 'Last Updater Name='+ bcolors.ENDC,req['history'][0]['updater']['author_name'],\ + '(',req['history'][0]['updater']['author_email'],')',\ + '\n'\ + + bcolors.ENDC +\ + 'McM View Link= https://cms-pdmv.cern.ch/mcm/requests?shown=2199023255551&prepid='+req['prepid'],\ + '\n'\ + 'McM Edit Link= https://cms-pdmv.cern.ch/mcm/edit?db_name=requests&prepid='+req['prepid'],\ + '' + if listAttr > 3: + print bcolors.YELLOW +\ + 'Member of chain(s)' + for current_chain in req['member_of_chain']: + query_chains = "member_of_chain="+current_chain + # print "req['member_of_chain'][0]",query_chains + temp = sys.stdout + f = open('/dev/null', 'w') + sys.stdout = f + chained_prepIds=getMcMlist(query_chains,False) + sys.stdout = temp + prepid1 = [] + for req1 in chained_prepIds: + prepid1.append(req1['prepid']) + print current_chain+" : "+ bcolors.ENDC+str(prepid1).strip('[]').replace("u'",'').replace("'","") + print bcolors.ENDC +\ + 'McM View Link= https://cms-pdmv.cern.ch/mcm/chained_requests?shown=4095&prepid='+current_chain,\ + ''+ bcolors.YELLOW + if listAttr > 4: + print bcolors.Green_like_Grass +'Fragment code=\n'+\ + bcolors.ENDC +\ + req['fragment'],\ + ''+ bcolors.ENDC + print bcolors.ENDC + + print '================================================================================\n\n',\ + +def getPrepIDList(query_string, getNew, getForValidation, getChain): + req_list = getMcMlist(query_string,True) event_sum = 0 out_list = [] @@ -77,53 +212,64 @@ def isSequential(lastID, currentID): current = currentID.split('-') if len(last) == 3 and len(current) == 3: - if last[0] == current[0] \ - and last[1] == current[1] \ + if last[0] == current[0] and last[1] == current[1] \ and int(last[2]) + 1 == int(current[2]): return True return False def printList(list, format): - arrow = "-" - comma = "," - if format == 1: - arrow = " ---> " - comma = ", " - elif format == 2: - arrow = " ---> " - comma = "
" - - lastID = "FIRST" - print_last = False - last_index = len(list) - 1 print "" - for i, PrepID in enumerate(list): - if isSequential(lastID, PrepID): - if i < last_index: - print_last = True - else: - sys.stdout.write("{0}{1}".format(arrow, PrepID)) - else: - if print_last: - sys.stdout.write("{0}{1}{2}{3}".format(arrow, lastID, comma, - PrepID)) - elif i > 0: - sys.stdout.write("{0}{1}".format(comma, PrepID)) + if format == 3: + for PrepID in list: + print PrepID + print "" + else: + arrow = "-" + comma = "," + if format == 1: + arrow = " ---> " + comma = ", " + elif format == 2: + arrow = " ---> " + comma = "
" + + lastID = "FIRST" + print_last = False + last_index = len(list) - 1 + for i, PrepID in enumerate(list): + if isSequential(lastID, PrepID): + if i < last_index: + print_last = True + else: + sys.stdout.write("{0}{1}".format(arrow, PrepID)) else: - sys.stdout.write("{0}".format(PrepID)) - print_last = False - lastID = PrepID - print "\n" + if print_last: + sys.stdout.write("{0}{1}{2}{3}".format(arrow, lastID, comma, + PrepID)) + elif i > 0: + sys.stdout.write("{0}{1}".format(comma, PrepID)) + else: + sys.stdout.write("{0}".format(PrepID)) + print_last = False + lastID = PrepID + print "" + + print "" return def main(): args = getArguments() # Setup flags and get arguments - list = getPrepIDList(args.query, args.getNew, args.getForValidation, - args.getChain) - printList(list, args.format) + if args.listAttr < 0: + list = getPrepIDList(args.query, args.getNew, args.getForValidation, + args.getChain) + printList(list, args.format) + else: + if args.noColors: + bcolors._noColor() + dict = getPrepIDListWithAttributes(args.query,args.listAttr) return diff --git a/makeTicket.py b/makeTicket.py new file mode 100755 index 0000000..66707a9 --- /dev/null +++ b/makeTicket.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python + +################################ +# +# makeTicket.py +# +# Script to create tickets for MCCM. +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sys +import argparse +import pprint +import mcmscripts_config +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * # Load class to access McM + + +def getArguments(): + parser = argparse.ArgumentParser( + description='Script to creat tickets in McM.') + + # Command line flags + parser.add_argument('-i', '--ids', dest='ids', help='List of PrepIDs.') + parser.add_argument('-p', '--pwg', action='store', dest='pwg', + default=mcmscripts_config.pwg, + help='Set PWG. Defaults to %(default)s. Change default in config.py') + parser.add_argument('-c', '--chain', action='store', dest='chain', + metavar='name', help='Set chain.') + parser.add_argument('-b', '--block', action='store', dest='block', type=int, + default=3, help='Set block.') + parser.add_argument('-r', '--repititions', action='store', + dest='repititions', type=int, default=1, + help='Number of chains.') + parser.add_argument('-s', '--staged', dest='staged', action='store', + type=int, default=0, help='Number of events needed.') + + args_ = parser.parse_args() + return args_ + + +def interrogate(): + loop = True + while loop: + chain = raw_input("Chain: ") + block = raw_input("Block: ") + repititions = raw_input("Repititions: ") + staged = raw_input("Staged: ") + + yn = raw_input("Are these acceptable? [y/n] ") + while True: + if yn.lower() in ["y", "yes"]: + loop = False + break + elif yn.lower() in ["n", "no"]: + break + else: + yn = raw_input('Please enter "yes" or "no". ') + + return [chain, block, repititions, staged] + + +def formatIDRange(pwg, campaign, first, last): + first = int(first) + last = int(last) + if first > last: + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) + sys.exit(3) + + return ['{0}-{1}-{2:05d}'.format(pwg, campaign, first), + '{0}-{1}-{2:05d}'.format(pwg, campaign, last)] + + +def parseIDList(compactList): + splitList = compactList.split(',') + requests = [] + for subList in splitList: + splitSubList = subList.split('-') + if len(splitSubList) == 3: + requests.append(subList) + elif len(splitSubList) == 4: + requests.append(formatIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[3])) + elif len(splitSubList) == 6: + if splitSubList[0] != splitSubList[3]: + print "Error: PrepID range must be for the same PWG." + sys.exit(2) + if splitSubList[1] != splitSubList[4]: + print "Error: PrepID range must be for the same campaign." + sys.exit(2) + requests.append(formatIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[5])) + else: + print "Error: Poorly formed PrepID list." + sys.exit(1) + return requests + + +def createTicket(requests, pwg, chain, block, repititions, staged): + mcm = restful(dev=False) + ticket = {'prepid': pwg, 'pwg': pwg, 'block': block, + 'chains': [chain], 'repetitions': repititions, + 'requests': requests + } + + pprint.pprint(ticket) + + # ticket = {'prepid': 'EXO', 'pwg': 'EXO', 'block': 2, + # 'chains': ['RunIISpring16DR80PU2016MiniAODv1'], 'repetitions': 1, + # 'requests': [['EXO-RunIISummer15GS-00435', 'EXO-RunIISummer15GS-00509'], ['EXO-RunIISummer15GS-06132', 'EXO-RunIISummer15GS-06139']] + # } + # ticket = {'prepid': 'EXO', 'pwg': 'EXO', 'block': 2, + # 'chains': ['RunIISpring16DR80PU2016MiniAODv1wmLHE'], 'repetitions': 1, + # 'notes': 'For B2G', + # 'requests': ['EXO-RunIIWinter15wmLHE-00288'] + # } + + answer = mcm.putA('mccms', ticket) + if answer['results']: + print "Created {0}".format(answer['prepid']) + else: + print "Failed" + + return + + +def modifyTicket(): + mcm = restful(dev=False) + ticket = mcm.getA('mccms', 'EXO-2016Mar30-00024') + #ticket['block'] = 2 + #ticket['chains'] = ['RunIISpring16DR80PU2016MiniAODv1'] + #ticket['chains'] = ['RunIISpring16DR80PU2016MiniAODv1wmLHE'] + #ticket['repetitions'] = 1 + #ticket['requests'] = [['EXO-RunIIWinter15wmLHE-01572', + pprint.pprint(ticket) + + # answer = mcm.updateA('mccms', ticket) + # print answer + # if answer['results']: + # print "Modified" + # else: + # print "Failed" + + return + + +def main(): + args = getArguments() # Setup flags and get arguments + + (args.chain, args.block, args.repititions, args.staged) = interrogate() + + requests = parseIDList(args.ids) + print requests + + createTicket(requests, args.pwg, args.chain, args.block, args.repititions, + args.staged) + return + + +if __name__ == '__main__': + main() diff --git a/manageRequests.py b/manageRequests.py index 21acc92..aeda9f5 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -16,25 +16,35 @@ import csv import pprint import time +import urllib2 +import mcmscripts_config sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import * # Load class to access McM from requestClass import * # Load class to store request information def getArguments(): - defaultPWG = 'XXX' # Change this line to your PWG, then -p flag is not needed - - parser = argparse.ArgumentParser(description='Create, modify, and clone McM requests.') + parser = argparse.ArgumentParser( + description='Create, modify, and clone McM requests.') # Command line flags parser.add_argument('file_in') - parser.add_argument('-c', '--campaign', action='store', dest='campaign', metavar='name', help='Set member_of_campaign.') - parser.add_argument('-p', '--pwg', action='store', dest='pwg', default=defaultPWG, help='Set PWG. Defaults to %(default)s. Change the variable defaultPWG to your PWG.') - parser.add_argument('-m', '--modify', action='store_true', dest='doModify', help='Modify existing requests. The CSV file must contain the PrepIds of the requests to be modified.') - parser.add_argument('--clone', action='store', dest='cloneId', default='', help='Clone an existing request by giving its PrepId') - parser.add_argument('-d', '--dry', action='store_true', dest='doDryRun', help='Dry run of result. Does not add requests to McM.') - parser.add_argument('--dev', action='store_true', dest='useDev', help='Use dev/test instance.') - parser.add_argument('-l', '--lhe', action='store_true', dest='isLHErequest', help='Check dataset when modifying requests. Fail and do not modify name if they conflict. Use for updating GS requests chained to wmLHE and pLHE requests.') - parser.add_argument('-v', '--version', action='version', version='%(prog)s v1.0') + parser.add_argument('-c', '--campaign', action='store', dest='campaign', + metavar='name', help='Set member_of_campaign.') + parser.add_argument('-p', '--pwg', action='store', dest='pwg', + default=mcmscripts_config.pwg, + help='Set PWG. Defaults to %(default)s. Change default in config.py') + parser.add_argument('-m', '--modify', action='store_true', dest='doModify', + help='Modify existing requests. The CSV file must contain the PrepIds of the requests to be modified.') + parser.add_argument('--clone', action='store', dest='cloneId', default='', + help='Clone an existing request by giving its PrepId') + parser.add_argument('-d', '--dry', action='store_true', dest='doDryRun', + help='Dry run of result. Does not add requests to McM.') + parser.add_argument('--dev', action='store_true', dest='useDev', + help='Use dev/test instance.') + parser.add_argument('-l', '--lhe', action='store_true', dest='isLHErequest', + help='Check dataset when modifying requests. Fail and do not modify name if they conflict. Use for updating GS requests chained to wmLHE and pLHE requests.') + parser.add_argument('-t', '--tags', action='append', dest='McMTags', + metavar='tags', help='Tags to append to request in McM.') args_ = parser.parse_args() return args_ @@ -42,15 +52,16 @@ def getArguments(): def checkFile(file_): # Check that CSV file exists if not os.path.isfile(file_): - print "Error: File %s does not exist." % file_ - print "Exiting with status 1." + print "Error: File {0} does not exist.".format(file_) sys.exit(1) def checkPWG(pwg_): - pwg_list = ['B2G','BPH','BTW','EGM','EWK','EXO','FSQ','FWD','HCA','HIG','HIN','JME','L1T','MUO','QCD','SMP','SUS','TAU','TOP','TRK','TSG'] + pwg_list = ['B2G', 'BPH', 'BTW', 'EGM', 'EWK', 'EXO', 'FSQ', 'FWD', 'HCA', + 'HIG', 'HIN', 'JME', 'L1T', 'MUO', 'QCD', 'SMP', 'SUS', 'TAU', + 'TOP', 'TRK', 'TSG'] # Check that PWG is valid if pwg_ not in pwg_list: - print "Error: %s is not a recognized PWG." % pwg_ + print "Error: {0} is not a recognized PWG.".format(pwg_) if pwg_ == 'XXX': print "Change the default value for flag -p to your PWG by modifying the variable defaultPWG on line 23." sys.stdout.write("Options are:") @@ -58,106 +69,141 @@ def checkPWG(pwg_): sys.stdout.write(" ") sys.stdout.write(iPWG) sys.stdout.write("\n") - print "Exiting with status 2." sys.exit(2) -def checkNotCreate(doModify_,cloneId_): +def checkNotCreate(doModify_, cloneId_): # Check that script isn't being asked to both modify and clone a request doClone = False - if cloneId_ != "": doClone = True + if cloneId_ != "": + doClone = True if doModify_ and doClone: print "Error: cannot both --modify and --clone." - print "Exiting with status 6." sys.exit(6) return doModify_ or doClone # Return variable to use in fillFields() -def exitDuplicateField(file_in_,field_): - print "Error: File %s contains multiple instances of the field %s" % (file_in_,field_) - print "Exiting with status 3." +def exitDuplicateField(file_in_, field_): + print "Error: File {0} contains multiple instances of the field {1}".format( + file_in_, field_) sys.exit(3) -def getFields(csvfile_,file_in_): +def getFields(csvfile_, file_in_): # List of indices for each field in CSV file list = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1] + -1, -1, -1, -1, -1, -1, -1] header = csv.reader(csvfile_).next() for ind, field in enumerate(header): - if field in ['Dataset name','Dataset Name','Dataset','dataset']: + field = field.lower() + if field in ['dataset name', 'dataset']: #ensure no duplicate fields - if list[0] > -1: exitDuplicateField(file_in_,"Dataset name") + if list[0] > -1: + exitDuplicateField(file_in_, "Dataset name") list[0] = ind - elif field in ['EOS','eos','Eos','MCDBID','mcdbid']: - if list[1] > -1: exitDuplicateField(file_in_,"EOS") + elif field in ['eos', 'mcdbid']: + if list[1] > -1: + exitDuplicateField(file_in_, "EOS") list[1] = ind - elif field in ['Cross section [pb]','Cross section','Cross section (pb)','Cross Section','Cross Section [pb]','Cross Section (pb)','CS','CS [pb]','CS (pb)','Xsec','Xsec [pb]','Xsec (pb)']: - if list[2] > -1: exitDuplicateField(file_in_,"Cross section") + elif field in ['cross section [pb]', 'cross section (pb)', + 'cross section', 'cs', 'cs [pb]', 'cs (pb)', 'xsec', + 'xsec [pb]', 'xsec (pb)']: + if list[2] > -1: + exitDuplicateField(file_in_, "Cross section [pb]") list[2] = ind - elif field in ['Total events','Total Events','Events','events','total events','Number of Events']: - if list[3] > -1: exitDuplicateField(file_in_,"Total events") + elif field in ['total events', 'events', 'number of events']: + if list[3] > -1: + exitDuplicateField(file_in_, "Total events") list[3] = ind - elif field in ['Fragment name','Fragment Name','Generator fragment name','Generator Fragment Name','Fragment','fragment']: - if list[4] > -1: exitDuplicateField(file_in_,"Fragment name") + elif field in ['fragment name', 'generator fragment name', 'fragment']: + if list[4] > -1: + exitDuplicateField(file_in_, "Fragment name") list[4] = ind - elif field in ['Time per event [s]','Time per event','Time per event (s)','Time per Event','Time per Event [s]','Time per Event (s)','Time','Time [s]','Time (s)','time','time [s]','time (s)']: - if list[5] > -1: exitDuplicateField(file_in_,"Time per event [s]") + elif field in ['time per event [s]', 'time per event', + 'time per event (s)', 'time', 'time [s]', 'time (s)']: + if list[5] > -1: + exitDuplicateField(file_in_, "Time per event [s]") list[5] = ind - elif field in ['Size per event [kB]','Size per event','Size per event (kB)','Size per Event','Size per Event [kB]','Size per Event (kB)','size','size [kB]','size (kB)']: - if list[6] > -1: exitDuplicateField(file_in_,"Size per event [kB]") + elif field in ['size per event [kb]', 'size per event', + 'size per event (kb)', 'size', 'size [kb]', 'size (kb)']: + if list[6] > -1: + exitDuplicateField(file_in_, "Size per event [kB]") list[6] = ind - elif field in ['Tag','tag','Fragment Tag','Fragment tag','fragment tag','sha','SHA','SHA-1','sha-1']: - if list[7] > -1: exitDuplicateField(file_in_,"Fragment tag") + elif field in ['tag', 'fragment tag', 'sha', 'sha-1']: + if list[7] > -1: + exitDuplicateField(file_in_, "Fragment tag") list[7] = ind - elif field in ['Generator','generator']: - if list[8] > -1: exitDuplicateField(file_in_,"Generator") + elif field in ['generator', 'generators']: + if list[8] > -1: + exitDuplicateField(file_in_, "Generator") list[8] = ind - elif field in ['Filter efficiency','FilterEfficiency','filter efficiency']: - if list[9] > -1: exitDuplicateField(file_in_,"Filter efficiency") + elif field in ['filter efficiency']: + if list[9] > -1: + exitDuplicateField(file_in_, "Filter efficiency") list[9] = ind - elif field in ['Filter efficiency error','Filter Efficiency Error','filter efficiency error']: - if list[10] > -1: exitDuplicateField(file_in_,"Filter efficiency error") + elif field in ['filter efficiency error']: + if list[10] > -1: + exitDuplicateField(file_in_, "Filter efficiency error") list[10] = ind - elif field in ['Match efficiency','Match Efficiency','match efficiency']: - if list[11] > -1: exitDuplicateField(file_in_,"Match efficiency") + elif field in ['match efficiency']: + if list[11] > -1: + exitDuplicateField(file_in_, "Match efficiency") list[11] = ind - elif field in ['Match efficiency error','Match Efficiency Error','match efficiency error']: - if list[12] > -1: exitDuplicateField(file_in_,"Match efficiency error") + elif field in ['match efficiency error']: + if list[12] > -1: + exitDuplicateField(file_in_, "Match efficiency error") list[12] = ind - elif field in ['PWG','pwg']: - if list[13] > -1: exitDuplicateField(file_in_,"PWG") + elif field in ['pwg']: + if list[13] > -1: exitDuplicateField(file_in_, "PWG") list[13] = ind - elif field in ['Campaign','campaign','Member of Campaign','Member of campaign','member of campaign']: - if list[14] > -1: exitDuplicateField(file_in_,"Member of campaign") + elif field in ['campaign', 'member of campaign']: + if list[14] > -1: + exitDuplicateField(file_in_, "Member of campaign") list[14] = ind - elif field in ['PrepId','PrepID','PREPID','prepid']: - if list[15] > -1: exitDuplicateField(file_in_,"PrepId") + elif field in ['prepid']: + if list[15] > -1: + exitDuplicateField(file_in_, "PrepID") list[15] = ind - elif field in ['Sequences customise','Sequences customize']: - if list[16] > -1: exitDuplicateField(file_in_,"Sequences customise") + elif field in ['sequences customise', 'sequences customize']: + if list[16] > -1: + exitDuplicateField(file_in_, "Sequences customise") list[16] = ind - elif field in ['Process string','Process String']: - if list[17] > -1: exitDuplicateField(file_in_,"Process string") + elif field in ['process string']: + if list[17] > -1: + exitDuplicateField(file_in_, "Process string") list[17] = ind - elif field in ['Gridpack location', 'Gridpack']: - if list[18] > -1: exitDuplicateField(file_in_,"Gridpack location") + elif field in ['gridpack location', 'gridpack']: + if list[18] > -1: + exitDuplicateField(file_in_, "Gridpack location") list[18] = ind - elif field in ['Gridpack cards URL', 'Cards URL', - 'Gridpack cards location', 'Cards location']: - if list[19] > -1: exitDuplicateField(file_in_,"Gridpack cards URL") + elif field in ['gridpack cards url', 'cards url', + 'gridpack cards location', 'cards location']: + if list[19] > -1: + exitDuplicateField(file_in_, "Gridpack cards URL") list[19] = ind - elif field in ['Notes', 'notes']: - if list[20] > -1: exitDuplicateField(file_in_,"Notes") + elif field in ['notes']: + if list[20] > -1: + exitDuplicateField(file_in_, "Notes") list[20] = ind - elif field in ['JobId', 'Local gridpack location', 'Local LHE']: + elif field in ['mcm tag', 'mcm tags']: + if list[21] > -1: + exitDuplicateField(file_in_, "McM tags") + list[21] = ind + elif field in ['sequences beamspot']: + if list[22] > -1: + exitDuplicateField(file_in_, "Sequences beamspot") + list[22] = ind + elif field in ['sequences magfield']: + if list[23] > -1: + exitDuplicateField(file_in_, "Sequences magField") + list[23] = ind + elif field in ['jobid', 'local gridpack location', 'local lhe', 'lhe']: continue else: - print "Error: The field %s is not valid." % field - print "Exiting with status 4." + print "Error: The field {0} is not valid.".format(field) sys.exit(4) return list def formatFragment(file_, campaign_): - if len(file_.split("/")) > 1: + if len(file_.split("/")) > 2: return file_ # 8TeV campaign elif campaign_ in ['Summer12']: @@ -165,38 +211,53 @@ def formatFragment(file_, campaign_): # 13 TeV campaigns elif campaign_ in ['Fall13', 'RunIIFall14GS', 'RunIIWinter15GS', 'RunIIWinter15wmLHE', 'RunIIWinter15pLHE', - 'RunIISummer15GS']: + 'RunIISummer15GS', 'RunIISummer15wmLHEGS']: return "Configuration/GenProduction/python/ThirteenTeV/"+file_ else: print "Error: Cannot determine energy of campaign {0}.".format(campaign_) sys.exit(5) -def createLHEProducer(gridpack, cards): +def createLHEProducer(gridpack, cards, fragment, tag): code = """import FWCore.ParameterSet.Config as cms externalLHEProducer = cms.EDProducer("ExternalLHEProducer", - args = cms.vstring('%s'), + args = cms.vstring('{0}'), nEvents = cms.untracked.uint32(5000), numberOfParameters = cms.uint32(1), outputFile = cms.string('cmsgrid_final.lhe'), scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh') -)""" % (gridpack) +)""".format(gridpack) if cards != "": code += """ # Link to cards: -# %s -""" % (cards) +# {0} +""".format(cards) + + if fragment != "" and tag != "": + gen_fragment_url = "https://raw.githubusercontent.com/cms-sw/genproductions/{0}/{1}".format( + tag, fragment.split("Configuration/GenProduction/")[1]) + gen_fragment = urllib2.urlopen(gen_fragment_url).read() + code += """ +{0} + +# Link to generator fragment: +# {1} +""".format(gen_fragment, gen_fragment_url) + return code -def fillFields(csvfile, fields, campaign, PWG, notCreate_): +def fillFields(csvfile, fields, campaign, PWG, notCreate_, McMTags): requests = [] # List containing request objects num_requests = 0 for row in csv.reader(csvfile): + if row[0].startswith("#"): + continue num_requests += 1 tmpReq = Request() - if fields[0] > -1: tmpReq.setDataSetName(row[fields[0]]) + if fields[0] > -1: + tmpReq.setDataSetName(row[fields[0]]) if fields[1] > -1: tmpReq.setMCDBID(row[fields[1]]) elif not notCreate_: @@ -205,17 +266,23 @@ def fillFields(csvfile, fields, campaign, PWG, notCreate_): tmpReq.setCS(row[fields[2]]) elif not notCreate_: tmpReq.setCS(1.0) - if fields[3] > -1: tmpReq.setEvts(row[fields[3]]) + if fields[3] > -1: + tmpReq.setEvts(row[fields[3]]) if fields[14] > -1: campaign = row[fields[14]] tmpReq.setCamp(campaign) elif campaign is not None: tmpReq.setCamp(campaign) - if fields[4] > -1: tmpReq.setFrag(formatFragment(row[fields[4]],campaign)) - if fields[5] > -1: tmpReq.setTime(row[fields[5]]) - if fields[6] > -1: tmpReq.setSize(row[fields[6]]) - if fields[7] > -1: tmpReq.setTag(row[fields[7]]) - if fields[8] > -1: tmpReq.setGen(row[fields[8]].split(" ")) # Multiple generators separated by spaces + if fields[4] > -1: + tmpReq.setFrag(formatFragment(row[fields[4]],campaign)) + if fields[5] > -1: + tmpReq.setTime(row[fields[5]]) + if fields[6] > -1: + tmpReq.setSize(row[fields[6]]) + if fields[7] > -1: + tmpReq.setTag(row[fields[7]]) + if fields[8] > -1: + tmpReq.setGen(row[fields[8]].split(" ")) # Multiple generators separated by spaces if fields[9] > -1: tmpReq.setFiltEff(row[fields[9]]) elif not notCreate_: @@ -244,93 +311,138 @@ def fillFields(csvfile, fields, campaign, PWG, notCreate_): tmpReq.setProcessString(row[fields[17]]) if fields[18] > -1: if fields[19] > -1: - tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], - row[fields[19]])) + if fields[4] > -1 and fields[7]: + tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], + row[fields[19]], + formatFragment(row[fields[4]], campaign), + row[fields[7]])) + else: + tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], + row[fields[19]], "", "")) else: - tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], "")) + tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], "", "", "")) if fields[20] > -1: tmpReq.setNotes(row[fields[20]]) + if fields[21] > -1: + tmpReq.setMcMTag(row[fields[21]].split(" ")) + elif McMTags is not None: + tmpReq.setMcMTag(McMTags) + if fields[22] > -1: + tmpReq.setSequencesBeamspot(row[fields[22]]) + if fields[23] > -1: + tmpReq.setSequencesMagField(row[fields[23]]) requests.append(tmpReq) return requests, num_requests def createRequests(requests, num_requests, doDryRun, useDev): # Create new requests based on campaign and PWG - mcm = restful( dev=useDev ) # Get McM connection + mcm = restful(dev=useDev) # Get McM connection if not doDryRun: - print "Adding %d requests to McM." % num_requests + print "Adding {0} requests to McM.".format(num_requests) else: - print "Dry run. %d requests will not be added to McM." % num_requests + print "Dry run. {0} requests will not be added to McM.".format( + num_requests) for reqFields in requests: if not reqFields.useCamp(): print "A campaign is needed for new requests." continue # Create new request's dictionary - new_req = {'pwg':reqFields.getPWG(),'member_of_campaign':reqFields.getCamp(),'mcdb_id':reqFields.getMCDBID()} + new_req = {'pwg': reqFields.getPWG(), + 'member_of_campaign': reqFields.getCamp(), + 'mcdb_id': reqFields.getMCDBID()} # Fill dictionary with fields - if reqFields.useDataSetName(): new_req['dataset_name'] = reqFields.getDataSetName() - if reqFields.useEvts(): new_req['total_events'] = reqFields.getEvts() - if reqFields.useFrag(): new_req['name_of_fragment'] = reqFields.getFrag() - if reqFields.useTag(): new_req['fragment_tag'] = reqFields.getTag() - if reqFields.useMcMFrag(): new_req['fragment'] = reqFields.getMcMFrag() - if reqFields.useTime(): new_req['time_event'] = reqFields.getTime() - if reqFields.useSize(): new_req['size_event'] = reqFields.getSize() - if reqFields.useGen(): new_req['generators'] = reqFields.getGen() - # Sequences might need to be added below with generator parameters - if reqFields.useSequencesCustomise(): new_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() - if reqFields.useProcessString(): new_req['process_string'] = reqFields.getProcessString() - if reqFields.useNotes(): new_req['notes'] = reqFields.getNotes() + if reqFields.useDataSetName(): + new_req['dataset_name'] = reqFields.getDataSetName() + if reqFields.useEvts(): + new_req['total_events'] = reqFields.getEvts() + if reqFields.useMcMFrag(): + new_req['fragment'] = reqFields.getMcMFrag() + else: + if reqFields.useFrag(): + new_req['name_of_fragment'] = reqFields.getFrag() + if reqFields.useTag(): + new_req['fragment_tag'] = reqFields.getTag() + if reqFields.useTime(): + new_req['time_event'] = reqFields.getTime() + if reqFields.useSize(): + new_req['size_event'] = reqFields.getSize() + if reqFields.useGen(): + new_req['generators'] = reqFields.getGen() + if reqFields.useProcessString(): + new_req['process_string'] = reqFields.getProcessString() + if reqFields.useNotes(): + new_req['notes'] = reqFields.getNotes() + if reqFields.useMcMTag(): + new_req['tags'] = reqFields.getMcMTag() if not doDryRun: answer = mcm.putA('requests', new_req) # Create request if answer['results']: # Cannot fill generator parameters while creating a new request # Modify newly created request with generator parameters - mod_req = mcm.getA('requests',answer['prepid']) # Get newly created request + # Get newly created request + mod_req = mcm.getA('requests', answer['prepid']) # Fill generator parameters - mod_req['generator_parameters'][0]['cross_section'] = reqFields.getCS() - mod_req['generator_parameters'][0]['filter_efficiency'] = reqFields.getFiltEff() - mod_req['generator_parameters'][0]['filter_efficiency_error'] = reqFields.getFiltEffErr() - mod_req['generator_parameters'][0]['match_efficiency'] = reqFields.getMatchEff() - mod_req['generator_parameters'][0]['match_efficiency_error'] = reqFields.getMatchEffErr() - update_answer = mcm.updateA('requests',mod_req) # Update request with generator parameters + mod_req['generator_parameters'][0]['cross_section'] \ + = reqFields.getCS() + mod_req['generator_parameters'][0]['filter_efficiency'] \ + = reqFields.getFiltEff() + mod_req['generator_parameters'][0]['filter_efficiency_error'] \ + = reqFields.getFiltEffErr() + mod_req['generator_parameters'][0]['match_efficiency'] \ + = reqFields.getMatchEff() + mod_req['generator_parameters'][0]['match_efficiency_error'] \ + = reqFields.getMatchEffErr() + if reqFields.useSequencesCustomise(): + mod_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() + if reqFields.useSequencesBeamspot(): + mod_req['sequences'][0]['beamspot'] = reqFields.getSequencesBeamspot() + if reqFields.useSequencesMagField(): + mod_req['sequences'][0]['magField'] = reqFields.getSequencesMagField() + # Update request with generator parameters and sequences + update_answer = mcm.updateA('requests', mod_req) if update_answer['results']: - print answer['prepid'],"created" + print "\033[0;32m{0} created\033[0;m".format(answer['prepid']) else: - print answer['prepid'],"created but generator parameters not set" + print "\033[0;33m{0} created but generator parameters not set\033[0;m".format( + answer['prepid']) else: if reqFields.useDataSetname(): - print reqFields.getDataSetName(),"failed to be created" + print "\033[0;31m{0} failed to be created\033[0;m".format( + reqFields.getDataSetName()) else: - print "A request has failed to be created" + print "\033[0;31mA request has failed to be created\033[0;m" else: if reqFields.useDataSetName(): - print reqFields.getDataSetName(),"not created" + print "\033[0;31m{0} not created\033[0;m".format( + reqFields.getDataSetName()) else: - print "request not created" + print "\033[0;31mrequest not created\033[0;m" pprint.pprint(new_req) def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): # Modify existing request based on PrepId - mcm = restful( dev=useDev ) # Get McM connection + mcm = restful(dev=useDev) # Get McM connection if not doDryRun: - print "Modifying %d requests to McM." % num_requests + print "Modifying {0} requests to McM.".format(num_requests) else: - print "Dry run. %d requests will not be modified in McM." % num_requests + print "Dry run. {0} requests will not be modified in McM.".format( + num_requests) for reqFields in requests: # Get request from McM if isLHErequest: if not reqFields.useDataSetName(): - print "\033[1;31mDataset name missing\033[1;m" + print "\033[0;31mDataset name missing\033[0;m" continue elif not reqFields.useCamp(): - print "\033[1;31m%s modification failed. Must provide campaign.\033[1;m" \ - % (reqFields.getDataSetName()) + print "\033[0;31m{0} modification failed. Must provide campaign.\033[0;m".format( + reqFields.getDataSetName()) continue - query_string = "dataset_name=%s&member_of_campaign=%s" \ - % (reqFields.getDataSetName(), reqFields.getCamp()) + query_string = "dataset_name={0}&member_of_campaign={1}".format( + reqFields.getDataSetName(), reqFields.getCamp()) failed_to_get = True for tries in range(3): time.sleep(0.1) @@ -339,110 +451,185 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): failed_to_get = False break if failed_to_get: - print "\033[1;31m%s modification failed. Could not get request from McM.\033[1;m"\ - % (reqFields.getDataSetName()) + print "\033[0;31m{0} modification failed. Could not get request from McM.\033[0;m".format( + reqFields.getDataSetName()) continue if len(mod_req_list) > 1: - print "\033[1;31m{0} modification failed. Too many requests match query.\033[1;m".format( + print "\033[0;31m{0} modification failed. Too many requests match query.\033[0;m".format( reqFields.getDataSetName()) continue elif len(mod_req_list) == 0: - print "\033[1;31m{0} modification failed. No requests match query.\033[1;m".format( + print "\033[0;31m{0} modification failed. No requests match query.\033[0;m".format( reqFields.getDataSetName()) continue mod_req = mod_req_list[0] else: if not reqFields.usePrepId(): - print "\033[1;31mPrepId is missing.\033[1;m" + print "\033[0;31mPrepId is missing.\033[0;m" continue time.sleep(0.1) - mod_req = mcm.getA('requests',reqFields.getPrepId()) + mod_req = mcm.getA('requests', reqFields.getPrepId()) if reqFields.useDataSetName() and not isLHErequest: mod_req['dataset_name'] = reqFields.getDataSetName() - if reqFields.useMCDBID(): mod_req['mcdb_id'] = reqFields.getMCDBID() - if reqFields.useEvts(): mod_req['total_events'] = reqFields.getEvts() - if reqFields.useFrag(): mod_req['name_of_fragment'] = reqFields.getFrag() - if reqFields.useTag(): mod_req['fragment_tag'] = reqFields.getTag() - if reqFields.useMcMFrag(): mod_req['fragment'] = reqFields.getMcMFrag() - if reqFields.useTime(): mod_req['time_event'] = reqFields.getTime() - if reqFields.useSize(): mod_req['size_event'] = reqFields.getSize() - if reqFields.useGen(): mod_req['generators'] = reqFields.getGen() - if (reqFields.useCS() or reqFields.useFiltEff() or reqFields.useFiltEffErr() or reqFields.useMatchEff() or reqFields.useMatchEffErr()) and mod_req['generator_parameters'] == []: - mod_req['generator_parameters'] = [{'match_efficiency_error': 0.0, 'match_efficiency': 1.0, 'filter_efficiency': 1.0, 'version': 0, 'cross_section': 1.0, 'filter_efficiency_error': 0.0}] - if reqFields.useCS(): mod_req['generator_parameters'][0]['cross_section'] = reqFields.getCS() - if reqFields.useFiltEff(): mod_req['generator_parameters'][0]['filter_efficiency'] = reqFields.getFiltEff() - if reqFields.useFiltEffErr(): mod_req['generator_parameters'][0]['filter_efficiency_error'] = reqFields.getFiltEffErr() - if reqFields.useMatchEff(): mod_req['generator_parameters'][0]['match_efficiency'] = reqFields.getMatchEff() - if reqFields.useMatchEffErr(): mod_req['generator_parameters'][0]['match_efficiency_error'] = reqFields.getMatchEffErr() - if reqFields.useSequencesCustomise(): mod_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() - if reqFields.useProcessString(): mod_req['process_string'] = reqFields.getProcessString() - if reqFields.useNotes(): mod_req['notes'] = reqFields.getNotes() + if reqFields.useMCDBID(): + mod_req['mcdb_id'] = reqFields.getMCDBID() + if reqFields.useEvts(): + mod_req['total_events'] = reqFields.getEvts() + if reqFields.useMcMFrag(): + mod_req['fragment'] = reqFields.getMcMFrag() + else: + if reqFields.useFrag(): + mod_req['name_of_fragment'] = reqFields.getFrag() + if reqFields.useTag(): + mod_req['fragment_tag'] = reqFields.getTag() + if reqFields.useTime(): + mod_req['time_event'] = reqFields.getTime() + if reqFields.useSize(): + mod_req['size_event'] = reqFields.getSize() + if reqFields.useGen(): + mod_req['generators'] = reqFields.getGen() + if (reqFields.useCS() or reqFields.useFiltEff() + or reqFields.useFiltEffErr() or reqFields.useMatchEff() + or reqFields.useMatchEffErr()) and mod_req['generator_parameters'] == []: + mod_req['generator_parameters'] = [{'match_efficiency_error': 0.0, + 'match_efficiency': 1.0, + 'filter_efficiency': 1.0, + 'version': 0, + 'cross_section': 1.0, + 'filter_efficiency_error': 0.0}] + if reqFields.useCS(): + mod_req['generator_parameters'][0]['cross_section'] \ + = reqFields.getCS() + if reqFields.useFiltEff(): + mod_req['generator_parameters'][0]['filter_efficiency'] \ + = reqFields.getFiltEff() + if reqFields.useFiltEffErr(): + mod_req['generator_parameters'][0]['filter_efficiency_error'] \ + = reqFields.getFiltEffErr() + if reqFields.useMatchEff(): + mod_req['generator_parameters'][0]['match_efficiency'] \ + = reqFields.getMatchEff() + if reqFields.useMatchEffErr(): + mod_req['generator_parameters'][0]['match_efficiency_error'] \ + = reqFields.getMatchEffErr() + if reqFields.useSequencesCustomise(): + mod_req['sequences'][0]['customise'] \ + = reqFields.getSequencesCustomise() + if reqFields.useSequencesBeamspot(): + mod_req['sequences'][0]['beamspot'] = reqFields.getSequencesBeamspot() + if reqFields.useSequencesMagField(): + mod_req['sequences'][0]['magField'] = reqFields.getSequencesMagField() + if reqFields.useProcessString(): + mod_req['process_string'] = reqFields.getProcessString() + if reqFields.useNotes(): + mod_req['notes'] = reqFields.getNotes() + if reqFields.useMcMTag(): + mod_req['tags'] += reqFields.getMcMTag() if not doDryRun: - answer = mcm.updateA('requests',mod_req) # Update request + answer = mcm.updateA('requests', mod_req) # Update request if answer['results']: if not isLHErequest: - print "\033[1;32m%s modified\033[1;m" % (reqFields.getPrepId()) + print "\033[0;32m{0} modified\033[0;m".format( + reqFields.getPrepId()) else: - print "\033[1;32m%s in %s modified\033[1;m" % (reqFields.getDataSetName(), - reqFields.getCamp()) + print "\033[0;32m{0} in {1} modified\033[0;m".format( + reqFields.getDataSetName(), reqFields.getCamp()) else: if not isLHErequest: - print "\033[1;31m",reqFields.getPrepId(),"failed to be modified\033[1;m" + print "\033[0;31m{0} failed to be modified\033[0;m".format( + reqFields.getPrepId()) else: - print "\033[1;31m",reqFields.getDataSetName(),"failed to be modified\033[1;m" + print "\033[0;31m{0} failed to be modified\033[0;m".format( + reqFields.getDataSetName()) else: if not isLHErequest: - print reqFields.getPrepId(),"not modified" + print "\033[0;31m{0} not modified\033[0;m".format( + reqFields.getPrepId()) pprint.pprint(mod_req) else: - print reqFields.getDataSetName(),"not modified" + print "\033[0;31m{0} not modified\033[0;m".format( + reqFields.getDataSetName()) pprint.pprint(mod_req) def cloneRequests(requests, num_requests, doDryRun, useDev, cloneId_): # Create new requests be cloning an old one based on PrepId - mcm = restful( dev=useDev ) # Get McM connection + mcm = restful(dev=useDev) # Get McM connection if not doDryRun: - print "Adding %d requests to McM using clone." % num_requests + print "Adding {0} requests to McM using clone.".format(num_requests) else: - print "Dry run. %d requests will not be added to McM using clone." % num_requests + print "Dry run. {0} requests will not be added to McM using clone.".format( + num_requests) for reqFields in requests: - clone_req = mcm.getA('requests',cloneId_) # Get request to clone - if reqFields.useDataSetName(): clone_req['dataset_name'] = reqFields.getDataSetName() - if reqFields.useMCDBID(): clone_req['mcdb_id'] = reqFields.getMCDBID() - if reqFields.useEvts(): clone_req['total_events'] = reqFields.getEvts() - if reqFields.useFrag(): clone_req['name_of_fragment'] = reqFields.getFrag() - if reqFields.useTag(): clone_req['fragment_tag'] = reqFields.getTag() - if reqFields.useMcMFrag(): clone_req['fragment'] = reqFields.getMcMFrag() - if reqFields.useTime(): clone_req['time_event'] = reqFields.getTime() - if reqFields.useSize(): clone_req['size_event'] = reqFields.getSize() - if reqFields.useGen(): clone_req['generators'] = reqFields.getGen() - if reqFields.useCS(): clone_req['generator_parameters'][0]['cross_section'] = reqFields.getCS() - if reqFields.useFiltEff(): clone_req['generator_parameters'][0]['filter_efficiency'] = reqFields.getFiltEff() - if reqFields.useFiltEffErr(): clone_req['generator_parameters'][0]['filter_efficiency_error'] = reqFields.getFiltEffErr() - if reqFields.useMatchEff(): clone_req['generator_parameters'][0]['match_efficiency'] = reqFields.getMatchEff() - if reqFields.useMatchEffErr(): clone_req['generator_parameters'][0]['match_efficiency_error'] = reqFields.getMatchEffErr() - if reqFields.useSequencesCustomise(): clone_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() - if reqFields.useProcessString(): clone_req['process_string'] = reqFields.getProcessString() - if reqFields.useNotes(): clone_req['notes'] = reqFields.getNotes() + clone_req = mcm.getA('requests', cloneId_) # Get request to clone + if reqFields.useDataSetName(): + clone_req['dataset_name'] = reqFields.getDataSetName() + if reqFields.useMCDBID(): + clone_req['mcdb_id'] = reqFields.getMCDBID() + if reqFields.useEvts(): + clone_req['total_events'] = reqFields.getEvts() + if reqFields.useMcMFrag(): + clone_req['fragment'] = reqFields.getMcMFrag() + else: + if reqFields.useFrag(): + clone_req['name_of_fragment'] = reqFields.getFrag() + if reqFields.useTag(): + clone_req['fragment_tag'] = reqFields.getTag() + if reqFields.useTime(): + clone_req['time_event'] = reqFields.getTime() + if reqFields.useSize(): + clone_req['size_event'] = reqFields.getSize() + if reqFields.useGen(): + clone_req['generators'] = reqFields.getGen() + if reqFields.useCS(): + clone_req['generator_parameters'][0]['cross_section'] \ + = reqFields.getCS() + if reqFields.useFiltEff(): + clone_req['generator_parameters'][0]['filter_efficiency'] \ + = reqFields.getFiltEff() + if reqFields.useFiltEffErr(): + clone_req['generator_parameters'][0]['filter_efficiency_error'] \ + = reqFields.getFiltEffErr() + if reqFields.useMatchEff(): + clone_req['generator_parameters'][0]['match_efficiency'] \ + = reqFields.getMatchEff() + if reqFields.useMatchEffErr(): + clone_req['generator_parameters'][0]['match_efficiency_error'] \ + = reqFields.getMatchEffErr() + if reqFields.useSequencesCustomise(): + clone_req['sequences'][0]['customise'] \ + = reqFields.getSequencesCustomise() + if reqFields.useSequencesBeamspot(): + clone_req['sequences'][0]['beamspot'] = reqFields.getSequencesBeamspot() + if reqFields.useSequencesMagField(): + clone_req['sequences'][0]['magField'] = reqFields.getSequencesMagField() + if reqFields.useProcessString(): + clone_req['process_string'] = reqFields.getProcessString() + if reqFields.useNotes(): + clone_req['notes'] = reqFields.getNotes() + if reqFields.useMcMTag(): + clone_req['tags'] += reqFields.getMcMTag() if not doDryRun: - answer = mcm.clone(cloneId_,clone_req) # Clone request + answer = mcm.clone(cloneId_, clone_req) # Clone request if answer['results']: - print answer['prepid'],"created using clone" + print "\033[0;32m{0} created using clone\033[0;m".format( + answer['prepid']) else: if reqFields.useDataSetName(): - print reqFields.getDataSetName(),"failed to be created using clone" + print "\033[0;31m{0} failed to be created using clone\033[0;m".format( + reqFields.getDataSetName()) else: - print "request failed to be created using clone" + print "\033[0;31mrequest failed to be created using clone\033[0;m" else: if reqFields.useDataSetName(): - print reqFields.getDataSetName(),"not created using clone" + print "\033[0;31m{0} not created using clone\033[0;m".format( + reqFields.getDataSetName()) else: - print "request not created using clone" + print "\033[0;31mrequest not created using clone\033[0;m" pprint.pprint(clone_req) def main(): @@ -450,23 +637,26 @@ def main(): checkPWG(args.pwg) # Make sure PWG is an actual PWG # Check that script is not asked to both modify and clone # Store variable that is true if script is asked to modify or clone - notCreate = checkNotCreate(args.doModify,args.cloneId) + notCreate = checkNotCreate(args.doModify, args.cloneId) checkFile(args.file_in) # Ensure CSV file exists if args.useDev: print "Using dev/test instance." - csvfile = open(args.file_in,'r') # Open CSV file - fields = getFields(csvfile,args.file_in) # Get list of field indices + csvfile = open(args.file_in, 'r') # Open CSV file + fields = getFields(csvfile, args.file_in) # Get list of field indices # Fill list of request objects with fields from CSV and get number of requests - requests, num_requests = fillFields(csvfile, fields, args.campaign, args.pwg, notCreate) + requests, num_requests = fillFields(csvfile, fields, args.campaign, + args.pwg, notCreate, args.McMTags) if args.doModify: # Modify existing requests - modifyRequests(requests, num_requests, args.doDryRun, args.useDev, args.isLHErequest) + modifyRequests(requests, num_requests, args.doDryRun, args.useDev, + args.isLHErequest) elif args.cloneId != "": # Create new requests using clone - cloneRequests(requests, num_requests, args.doDryRun, args.useDev, args.cloneId) + cloneRequests(requests, num_requests, args.doDryRun, args.useDev, + args.cloneId) else: # Create new requests createRequests(requests, num_requests, args.doDryRun, args.useDev) diff --git a/mcmscripts_config.py b/mcmscripts_config.py new file mode 100644 index 0000000..2b1403c --- /dev/null +++ b/mcmscripts_config.py @@ -0,0 +1,14 @@ +# Physics working group for manageRequests.py +# Options: B2G BPH BTW EGM EWK EXO FSQ FWD HCA HIG HIN JME L1T MUO QCD SMP SUS +# TAU TOP TRK TSG +pwg = 'XXX' + +# McM username +username = 'inewton' + +# Automatically remove color when piping or redirecting output +auto_bw = False + +# Database +database_location = 'MC_Requests.db' +html_location = '' diff --git a/requestClass.py b/requestClass.py index d40b8f3..a430812 100644 --- a/requestClass.py +++ b/requestClass.py @@ -28,9 +28,12 @@ def __init__(self): self.useCamp_ = False self.usePrepId_ = False self.useSequencesCustomise_ = False + self.useSequencesBeamspot_ = False + self.useSequencesMagField_ = False self.useProcessString_ = False self.useJobID_ = False self.useNotes_ = False + self.useMcMTag_ = False def setDataSetName(self,x): self.DataSetName_ = x @@ -86,6 +89,12 @@ def setPrepId(self,x): def setSequencesCustomise(self,x): self.SequencesCustomise_ = x self.useSequencesCustomise_ = True + def setSequencesBeamspot(self,x): + self.SequencesBeamspot_ = x + self.useSequencesBeamspot_ = True + def setSequencesMagField(self,x): + self.SequencesMagField_ = x + self.useSequencesMagField_ = True def setProcessString(self,x): self.ProcessString_ = x self.useProcessString_ = True @@ -95,6 +104,9 @@ def setJobID(self,x): def setNotes(self, x): self.Notes_ = x self.useNotes_ = True + def setMcMTag(self, x): + self.McMTag_ = x + self.useMcMTag_ = True def getDataSetName(self): return self.DataSetName_ @@ -132,12 +144,18 @@ def getPrepId(self): return self.PrepId_ def getSequencesCustomise(self): return self.SequencesCustomise_ + def getSequencesBeamspot(self): + return self.SequencesBeamspot_ + def getSequencesMagField(self): + return self.SequencesMagField_ def getProcessString(self): return self.ProcessString_ def getJobID(self): return self.JobID_ def getNotes(self): return self.Notes_ + def getMcMTag(self): + return self.McMTag_ def useDataSetName(self): return self.useDataSetName_ @@ -175,9 +193,15 @@ def usePrepId(self): return self.usePrepId_ def useSequencesCustomise(self): return self.useSequencesCustomise_ + def useSequencesBeamspot(self): + return self.useSequencesBeamspot_ + def useSequencesMagField(self): + return self.useSequencesMagField_ def useProcessString(self): return self.useProcessString_ def useJobID(self): return self.useJobID_ def useNotes(self): return self.useNotes_ + def useMcMTag(self): + return self.useMcMTag_ diff --git a/testRequests.py b/testRequests.py index 90ceb6e..dc3e336 100755 --- a/testRequests.py +++ b/testRequests.py @@ -32,8 +32,6 @@ def getArguments(): parser.add_argument('-o', '--output', dest='output', default='test.csv', help='Output CSV file') parser.add_argument('-n', dest='nEvents', help='Number of events to test.') - parser.add_argument('-v', '--version', action='version', - version='%(prog)s v1.0') args_ = parser.parse_args() return args_ @@ -43,14 +41,13 @@ def fillIDRange(pwg, campaign, first, last): last = int(last) requests = [] if first > last: - print "Error: PrepID range out of order. %s-%s-%05d > %s-%s-%05d" % ( - pwg, campaign, first, pwg, campaign, last) - print "Exiting with status 4." + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) sys.exit(4) for number in range(first, last+1): tmpReq = Request() - tmpReq.setPrepId("%s-%s-%05d" % (pwg, campaign, number)) + tmpReq.setPrepId("{0}-{1}-{2:05d}".format(pwg, campaign, number)) requests.append(tmpReq) return requests @@ -69,11 +66,9 @@ def parseIDList(compactList): elif len(splitSubList) == 6: if splitSubList[0] != splitSubList[3]: print "Error: PrepID range must be for the same PWG." - print "Exiting with status 4" sys.exit(4) if splitSubList[1] != splitSubList[4]: print "Error: PrepID range must be for the same campaign." - print "Exiting with status 4" sys.exit(4) requests = requests + fillIDRange(splitSubList[0], splitSubList[1], splitSubList[2], splitSubList[5]) @@ -91,42 +86,42 @@ def getTestScript(PrepID, nEvents): get_test = "" if nEvents is None: get_test = "curl -s --insecure \ -https://cms-pdmv.cern.ch/mcm/public/restapi/%s/get_test/%s -o %s.sh" % ( +https://cms-pdmv.cern.ch/mcm/public/restapi/{0}/get_test/{1} -o {2}.sh".format( request_type, PrepID, PrepID) else: + # add "/N" to end of URL to get N events get_test = "curl -s --insecure \ -https://cms-pdmv.cern.ch/mcm/public/restapi/%s/get_test/%s/%s -o %s.sh" % ( +https://cms-pdmv.cern.ch/mcm/public/restapi/{0}/get_test/{1}/{2} -o {3}.sh".format( request_type, PrepID, nEvents, PrepID) - # add "/N" to end of URL to get N events print get_test subprocess.call(get_test, shell=True) if request_type == "chained_requests" and nEvents is not None: - filename = "%s.sh" % (PrepID) - tmpfilename = "tmp%s.sh" % (PrepID) + filename = "{0}.sh".format(PrepID) + tmpfilename = "tmp{0}.sh".format(PrepID) inputfile = open(filename, 'r') outputfile = open(tmpfilename, 'w') for line in inputfile: - outline = re.sub('(.*--eventcontent LHE.*-n) \d*( .*)', r'\1 %s\2' - % (nEvents), line) - outline = re.sub('(.*--eventcontent DQM.*-n) \d*( .*)', r'\1 %s\2' - % (nEvents), outline) - outline = re.sub('(.*--eventcontent RAWSIM.*-n) \d*( .*)', r'\1 %s\2' - % (nEvents), outline) + outline = re.sub('(.*--eventcontent LHE.*-n) \d*( .*)', + r'\1 {0}\2'.format(nEvents), line) + outline = re.sub('(.*--eventcontent DQM.*-n) \d*( .*)', + r'\1 {0}\2'.format(nEvents), outline) + outline = re.sub('(.*--eventcontent RAWSIM.*-n) \d*( .*)', + r'\1 {0}\2'.format(nEvents), outline) outputfile.write(outline) inputfile.close() outputfile.close() os.rename(tmpfilename, filename) - subprocess.call("chmod 755 %s.sh" % (PrepID), shell=True) + subprocess.call("chmod 755 {0}.sh".format(PrepID), shell=True) return def submitToBatch(PrepId): - batch_command = "bsub -q 8nh %s.sh" % (PrepId) + batch_command = "bsub -q 8nh {0}.sh".format(PrepId) print batch_command output = subprocess.Popen(batch_command, stdout=subprocess.PIPE, shell=True).communicate()[0] - match = re.match('Job <(\d*)> is',output) + match = re.match('Job <(\d*)> is', output) jobID = match.group(1) return jobID @@ -137,7 +132,7 @@ def createTest(compactPrepIDList, outputFile, nEvents): csvfile.writerow(['PrepId', 'JobId', 'Time per event [s]', 'Size per event [kB]']) - print "Testing %d requests" % (len(requests)) + print "Testing {0} requests".format(len(requests)) for req in requests: getTestScript(req.getPrepId(), nEvents) jobID = submitToBatch(req.getPrepId()) @@ -147,7 +142,7 @@ def createTest(compactPrepIDList, outputFile, nEvents): csvfile.writerow([req.getPrepId(), req.getJobID(), "", ""]) else: mcm = restful(dev=False) # Get McM connection - mcm_req = mcm.getA('chained_requests',req.getPrepId()) + mcm_req = mcm.getA('chained_requests', req.getPrepId()) wmLHEPrepId = mcm_req['chain'][0] GSPrepId = mcm_req['chain'][1] csvfile.writerow([wmLHEPrepId, req.getJobID(), "", ""]) @@ -155,104 +150,127 @@ def createTest(compactPrepIDList, outputFile, nEvents): return def exitDuplicateField(file_in_,field_): - print "Error: File %s contains multiple instances of the field %s" % (file_in_,field_) - print "Exiting with status 5." + print "Error: File {0} contains multiple instances of the field {1}".format( + file_in_, field_) sys.exit(5) def getFields(csvfile): # List of indices for each field in CSV file list = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1] + -1, -1, -1, -1, -1, -1, -1, -1] header = csv.reader(csvfile).next() for ind, field in enumerate(header): - if field in ['Dataset name', 'Dataset Name', 'Dataset', 'dataset']: + field = field.lower() + if field in ['dataset name', 'dataset']: #ensure no duplicate fields - if list[0] > -1: exitDuplicateField(file_in_, "Dataset name") + if list[0] > -1: + exitDuplicateField(file_in_, "Dataset name") list[0] = ind - elif field in ['EOS', 'eos', 'Eos', 'MCDBID', 'mcdbid']: - if list[1] > -1: exitDuplicateField(file_in_, "EOS") + elif field in ['eos', 'mcdbid']: + if list[1] > -1: + exitDuplicateField(file_in_, "EOS") list[1] = ind - elif field in ['Cross section [pb]', 'Cross section', - 'Cross section (pb)', 'Cross Section', - 'Cross Section [pb]', 'Cross Section (pb)', 'CS', - 'CS [pb]', 'CS (pb)', 'Xsec', 'Xsec [pb]', 'Xsec (pb)']: - if list[2] > -1: exitDuplicateField(file_in_, "Cross section") + elif field in ['cross section [pb]', 'cross section', + 'cross section (pb)', 'cs', 'cs [pb]', 'cs (pb)', 'xsec', + 'xsec [pb]', 'xsec (pb)']: + if list[2] > -1: + exitDuplicateField(file_in_, "Cross section [pb]") list[2] = ind - elif field in ['Total events', 'Total Events', 'Events', 'events', - 'total events', 'Number of Events']: - if list[3] > -1: exitDuplicateField(file_in_, "Total events") + elif field in ['total events', 'events', 'number of events']: + if list[3] > -1: + exitDuplicateField(file_in_, "Total events") list[3] = ind - elif field in ['Fragment name', 'Fragment Name', - 'Generator fragment name', 'Generator Fragment Name', - 'Fragment', 'fragment']: - if list[4] > -1: exitDuplicateField(file_in_, "Fragment name") + elif field in ['fragment name', 'generator fragment name', 'fragment']: + if list[4] > -1: + exitDuplicateField(file_in_, "Fragment name") list[4] = ind - elif field in ['Time per event [s]', 'Time per event', - 'Time per event (s)', 'Time per Event', - 'Time per Event [s]', 'Time per Event (s)', 'Time', - 'Time [s]', 'Time (s)', 'time', 'time [s]', 'time (s)']: - if list[5] > -1: exitDuplicateField(file_in_, "Time per event [s]") + elif field in ['time per event [s]', 'time per event', + 'time per event (s)', 'time', 'time [s]', 'time (s)']: + if list[5] > -1: + exitDuplicateField(file_in_, "Time per event [s]") list[5] = ind - elif field in ['Size per event [kB]', 'Size per event', - 'Size per event (kB)', 'Size per Event', - 'Size per Event [kB]', 'Size per Event (kB)', - 'size', 'size [kB]', 'size (kB)']: - if list[6] > -1: exitDuplicateField(file_in_, "Size per event [kB]") + elif field in ['size per event [kb]', 'size per event', + 'size per event (kb)', 'size', 'size [kb]', 'size (kb)']: + if list[6] > -1: + exitDuplicateField(file_in_, "Size per event [kB]") list[6] = ind - elif field in ['Tag', 'tag', 'Fragment Tag', 'Fragment tag', - 'fragment tag', 'sha', 'SHA', 'SHA-1', 'sha-1']: - if list[7] > -1: exitDuplicateField(file_in_, "Fragment tag") + elif field in ['tag', 'fragment tag', 'sha', 'sha-1']: + if list[7] > -1: + exitDuplicateField(file_in_, "Fragment tag") list[7] = ind - elif field in ['Generator', 'generator']: - if list[8] > -1: exitDuplicateField(file_in_, "Generator") + elif field in ['generator', 'generators']: + if list[8] > -1: + exitDuplicateField(file_in_, "Generator") list[8] = ind - elif field in ['Filter efficiency', 'FilterEfficiency', - 'filter efficiency']: - if list[9] > -1: exitDuplicateField(file_in_, "Filter efficiency") + elif field in ['filter efficiency']: + if list[9] > -1: + exitDuplicateField(file_in_, "Filter efficiency") list[9] = ind - elif field in ['Filter efficiency error', 'Filter Efficiency Error', - 'filter efficiency error']: - if list[10] > -1: exitDuplicateField(file_in_, - "Filter efficiency error") + elif field in ['filter efficiency error']: + if list[10] > -1: + exitDuplicateField(file_in_, "Filter efficiency error") list[10] = ind - elif field in ['Match efficiency', 'Match Efficiency', - 'match efficiency']: - if list[11] > -1: exitDuplicateField(file_in_, "Match efficiency") + elif field in ['match efficiency']: + if list[11] > -1: + exitDuplicateField(file_in_, "Match efficiency") list[11] = ind - elif field in ['Match efficiency error', 'Match Efficiency Error', - 'match efficiency error']: - if list[12] > -1: exitDuplicateField(file_in_, - "Match efficiency error") + elif field in ['match efficiency error']: + if list[12] > -1: + exitDuplicateField(file_in_, "Match efficiency error") list[12] = ind - elif field in ['PWG','pwg']: - if list[13] > -1: exitDuplicateField(file_in_, "PWG") + elif field in ['pwg']: + if list[13] > -1: + exitDuplicateField(file_in_, "PWG") list[13] = ind - elif field in ['Campaign', 'campaign', 'Member of Campaign', - 'Member of campaign', 'member of campaign']: - if list[14] > -1: exitDuplicateField(file_in_, "Member of campaign") + elif field in ['campaign', 'member of campaign']: + if list[14] > -1: + exitDuplicateField(file_in_, "Member of campaign") list[14] = ind - elif field in ['PrepId', 'PrepID', 'PREPID', 'prepid']: - if list[15] > -1: exitDuplicateField(file_in_, "PrepId") + elif field in ['prepid']: + if list[15] > -1: + exitDuplicateField(file_in_, "PrepID") list[15] = ind - elif field in ['Sequences customise', 'Sequences customize']: - if list[16] > -1: exitDuplicateField(file_in_, "Sequences customise") + elif field in ['sequences customise', 'sequences customize']: + if list[16] > -1: + exitDuplicateField(file_in_, "Sequences customise") list[16] = ind - elif field in ['Process string', 'Process String']: - if list[17] > -1: exitDuplicateField(file_in_, "Process string") + elif field in ['process string']: + if list[17] > -1: + exitDuplicateField(file_in_, "Process string") list[17] = ind - elif field in ['Gridpack location', 'Gridpack']: - if list[18] > -1: exitDuplicateField(file_in_, "Gridpack location") + elif field in ['gridpack location', 'gridpack']: + if list[18] > -1: + exitDuplicateField(file_in_, "Gridpack location") list[18] = ind - elif field in ['Gridpack cards URL', 'Cards URL', - 'Gridpack cards location', 'Cards location']: - if list[19] > -1: exitDuplicateField(file_in_, "Gridpack cards URL") + elif field in ['gridpack cards url', 'cards url', + 'gridpack cards location', 'cards location']: + if list[19] > -1: + exitDuplicateField(file_in_, "Gridpack cards URL") list[19] = ind - elif field in ['JobId']: - if list[20] > -1: exitDuplicateField(file_in_, "JobId") + elif field in ['jobid']: + if list[20] > -1: + exitDuplicateField(file_in_, "JobId") list[20] = ind + elif field in ['notes']: + if list[21] > -1: + exitDuplicateField(file_in_, "Notes") + list[21] = ind + elif field in ['mcm tag', 'mcm tags']: + if list[22] > -1: + exitDuplicateField(file_in_, "McM tags") + list[22] = ind + elif field in ['sequences beamspot']: + if list[23] > -1: + exitDuplicateField(file_in_, "Sequences beamspot") + list[23] = ind + elif field in ['sequences magField']: + if list[24] > -1: + exitDuplicateField(file_in_, "Sequences magField") + list[24] = ind + elif field in ['local gridpack location', 'Local lhe', 'lhe']: + continue else: - print "Error: The field %s is not valid." % (field) - print "Exiting with status 5." + print "Error: The field {0} is not valid.".format(field) sys.exit(6) return list @@ -261,24 +279,31 @@ def fillFields(csvfile, fields): requests = [] # List containing request objects num_requests = 0 for row in csv.reader(csvfile): + if row[0].startswith("#"): + continue num_requests += 1 tmpReq = Request() - if fields[0] > -1: tmpReq.setDataSetName(row[fields[0]]) + if fields[0] > -1: + tmpReq.setDataSetName(row[fields[0]]) if fields[1] > -1: tmpReq.setMCDBID(row[fields[1]]) if fields[2] > -1: tmpReq.setCS(row[fields[2]]) - if fields[3] > -1: tmpReq.setEvts(row[fields[3]]) + if fields[3] > -1: + tmpReq.setEvts(row[fields[3]]) if fields[14] > -1: campaign = row[fields[14]] tmpReq.setCamp(campaign) - if fields[4] > -1: tmpReq.setFrag(formatFragment(row[fields[4]],campaign)) + if fields[4] > -1: + tmpReq.setFrag(formatFragment(row[fields[4]],campaign)) if fields[5] > -1 and row[fields[5]] != "": tmpReq.setTime(row[fields[5]]) if fields[6] > -1 and row[fields[6]] != "": tmpReq.setSize(row[fields[6]]) - if fields[7] > -1: tmpReq.setTag(row[fields[7]]) - if fields[8] > -1: tmpReq.setGen(row[fields[8]].split(" ")) # Multiple generators separated by spaces + if fields[7] > -1: + tmpReq.setTag(row[fields[7]]) + if fields[8] > -1: + tmpReq.setGen(row[fields[8]].split(" ")) # Multiple generators separated by spaces if fields[9] > -1: tmpReq.setFiltEff(row[fields[9]]) if fields[10] > -1: @@ -303,6 +328,14 @@ def fillFields(csvfile, fields): tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], "")) if fields[20] > -1: tmpReq.setJobID(row[fields[20]]) + if fields[21] > -1: + tmpReq.setNotes(row[fields[21]]) + if fields[22] > -1: + tmpReq.setMcMTag(row[fields[22]].split(" ")) + if fields[23] > -1: + tmpReq.setSequencesBeamspot(row[fields[23]]) + if fields[24] > -1: + tmpReq.setSequencesMagField(row[fields[24]]) requests.append(tmpReq) return requests, num_requests @@ -331,7 +364,8 @@ def getTimeSizeFromFile(stdoutFile, iswmLHE): if match is not None: nEvents = float(match.group(1)) continue - match = re.match(' ', line) + match = re.match(' ', + line) if match is not None: totalSize = float(match.group(1)) continue @@ -352,7 +386,7 @@ def getTimeSize(requests): number_complete = 0 for req in requests: if not req.useTime() or not req.useSize(): - stdoutFile = "LSFJOB_%s/STDOUT" % (req.getJobID()) + stdoutFile = "LSFJOB_{0}/STDOUT".format(req.getJobID()) if os.path.exists(stdoutFile): number_complete += 1 iswmLHE = False @@ -367,9 +401,9 @@ def getTimeSize(requests): number_complete += 1 if number_complete == len(requests): - print "Extracted info for all %d requests." % (len(requests)) + print "Extracted info for all {0} requests.".format(len(requests)) else: - print "Extracted info for %d of %d requests. %d requests remain." % ( + print "Extracted info for {0} of {1} requests. {2} requests remain.".format( number_complete, len(requests), len(requests) - number_complete) return @@ -382,7 +416,7 @@ def extractTest(csvFile): getTimeSize(requests) - csvfile = open(csvFile,'w') + csvfile = open(csvFile, 'w') rewriteCSVFile(csvfile, requests) return @@ -393,7 +427,7 @@ def main(): print "Error: Cannot use both -i and -f." sys.exit(1) elif args.ids: - createTest(args.ids,args.output,args.nEvents) + createTest(args.ids, args.output, args.nEvents) elif args.csv: extractTest(args.csv) else: diff --git a/validateChains.py b/validateChains.py index 52631e6..491dae4 100755 --- a/validateChains.py +++ b/validateChains.py @@ -34,8 +34,8 @@ def fillIDRange(pwg, campaign, first, last): last = int(last) chains = [] if first > last: - print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {3}-{4}-{5:05d}".format( - pwg, campaign, first, pwg, campaign, last) + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) sys.exit(1) for number in range(first, last+1): @@ -72,21 +72,22 @@ def parseIDList(compactList): def validate(chains): mcm = restful(dev=False) - print "Validating %d chained requests" % (len(chains)) + print "Validating {0} chained requests".format(len(chains)) for PrepID in chains: - url = 'restapi/chained_requests/test/%s' % (PrepID) + url = 'restapi/chained_requests/test/{0}'.format(PrepID) chain_output = mcm.get(url) if chain_output['results']: - print "%s validating" % (PrepID) + print "{0} validating".format(PrepID) else: - print "%s will not be validated, due to the following reason: \n %s" % (PrepID,chain_output['message']) + print "{0} will not be validated, due to the following reason:\n{1}".format( + PrepID, chain_output['message']) def main(): - args = getArguments() # Setup flags and get arguments - chains = parseIDList(args.ids) # Get list of chains and check args - validate(chains) # Tell McM to validate chains + args = getArguments() # Setup flags and get arguments + chains = parseIDList(args.ids) # Get list of chains and check args + validate(chains) # Tell McM to validate chains if __name__ == '__main__':