From 738d96a391739ea739685dc54e2e24c3366e9e82 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 21 Oct 2015 22:19:18 +0200 Subject: [PATCH 01/64] Standardize style --- checkRequests.py | 8 +- getRequests.py | 20 +- manageRequests.py | 469 ++++++++++++++++++++++++++++++---------------- testRequests.py | 147 ++++++++------- validateChains.py | 19 +- 5 files changed, 410 insertions(+), 253 deletions(-) diff --git a/checkRequests.py b/checkRequests.py index d415792..9670a9e 100755 --- a/checkRequests.py +++ b/checkRequests.py @@ -36,9 +36,8 @@ def fillIDRange(pwg, campaign, first, last): last = int(last) requests = [] if first > last: - print "Error: PrepID range out of order. %s-%s-%05d > %s-%s-%05d" % ( - pwg, campaign, first, pwg, campaign, last) - print "Exiting with status 4." + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) sys.exit(4) for number in range(first, last+1): @@ -59,17 +58,14 @@ def parseIDList(compactList): elif len(splitSubList) == 6: if splitSubList[0] != splitSubList[3]: print "Error: PrepID range must be for the same PWG." - print "Exiting with status 4" sys.exit(4) if splitSubList[1] != splitSubList[4]: print "Error: PrepID range must be for the same campaign." - print "Exiting with status 4" sys.exit(4) requests = requests + fillIDRange(splitSubList[0], splitSubList[1], splitSubList[2], splitSubList[5]) else: print "Error: Poorly formed PrepID list." - print "Exiting with status 3." sys.exit(3) return requests diff --git a/getRequests.py b/getRequests.py index 8e4e835..aa20911 100755 --- a/getRequests.py +++ b/getRequests.py @@ -22,13 +22,17 @@ def getArguments(): - parser = argparse.ArgumentParser(description='Get a list of PrepIDs from McM based on a query.') + parser = argparse.ArgumentParser( + description='Get a list of PrepIDs from McM based on a query.') # Command line flags parser.add_argument('query') - parser.add_argument('-n', action='store_true', dest='getNew', help='Only get requests with unmodified time and size per event.') - parser.add_argument('-v', action='store_true', dest='getForValidation', help='Only get requests with positive time and size per event.') - parser.add_argument('-c', action='store_true', dest='getChain', help='Return PrepID of chain.') + parser.add_argument('-n', action='store_true', dest='getNew', + help='Only get requests with unmodified time and size per event.') + parser.add_argument('-v', action='store_true', dest='getForValidation', + help='Only get requests with positive time and size per event.') + parser.add_argument('-c', action='store_true', dest='getChain', + help='Return PrepID of chain.') parser.add_argument('-f', dest='format', type=int, default=0, help='Format of output. 0 (default) = input for scripts, 1 = human-readable, 2 = HTML') @@ -39,14 +43,13 @@ def getArguments(): def checkFile(file_): # Check that CSV file exists if not os.path.isfile(file_): - print "Error: File %s does not exist." % file_ - print "Exiting with status 1." + print "Error: File {0} does not exist.".format(file_) sys.exit(1) def getPrepIDList(query_string, getNew, getForValidation, getChain): useDev = False - mcm = restful( dev=useDev ) # Get McM connection + mcm = restful(dev=useDev) # Get McM connection print query_string req_list = mcm.getA('requests', query=query_string) @@ -77,8 +80,7 @@ def isSequential(lastID, currentID): current = currentID.split('-') if len(last) == 3 and len(current) == 3: - if last[0] == current[0] \ - and last[1] == current[1] \ + if last[0] == current[0] and last[1] == current[1] \ and int(last[2]) + 1 == int(current[2]): return True return False diff --git a/manageRequests.py b/manageRequests.py index 21acc92..17f3d83 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -23,18 +23,26 @@ def getArguments(): defaultPWG = 'XXX' # Change this line to your PWG, then -p flag is not needed - parser = argparse.ArgumentParser(description='Create, modify, and clone McM requests.') + parser = argparse.ArgumentParser( + description='Create, modify, and clone McM requests.') # Command line flags parser.add_argument('file_in') - parser.add_argument('-c', '--campaign', action='store', dest='campaign', metavar='name', help='Set member_of_campaign.') - parser.add_argument('-p', '--pwg', action='store', dest='pwg', default=defaultPWG, help='Set PWG. Defaults to %(default)s. Change the variable defaultPWG to your PWG.') - parser.add_argument('-m', '--modify', action='store_true', dest='doModify', help='Modify existing requests. The CSV file must contain the PrepIds of the requests to be modified.') - parser.add_argument('--clone', action='store', dest='cloneId', default='', help='Clone an existing request by giving its PrepId') - parser.add_argument('-d', '--dry', action='store_true', dest='doDryRun', help='Dry run of result. Does not add requests to McM.') - parser.add_argument('--dev', action='store_true', dest='useDev', help='Use dev/test instance.') - parser.add_argument('-l', '--lhe', action='store_true', dest='isLHErequest', help='Check dataset when modifying requests. Fail and do not modify name if they conflict. Use for updating GS requests chained to wmLHE and pLHE requests.') - parser.add_argument('-v', '--version', action='version', version='%(prog)s v1.0') + parser.add_argument('-c', '--campaign', action='store', dest='campaign', + metavar='name', help='Set member_of_campaign.') + parser.add_argument('-p', '--pwg', action='store', dest='pwg', + default=defaultPWG, + help='Set PWG. Defaults to %(default)s. Change the variable defaultPWG to your PWG.') + parser.add_argument('-m', '--modify', action='store_true', dest='doModify', + help='Modify existing requests. The CSV file must contain the PrepIds of the requests to be modified.') + parser.add_argument('--clone', action='store', dest='cloneId', default='', + help='Clone an existing request by giving its PrepId') + parser.add_argument('-d', '--dry', action='store_true', dest='doDryRun', + help='Dry run of result. Does not add requests to McM.') + parser.add_argument('--dev', action='store_true', dest='useDev', + help='Use dev/test instance.') + parser.add_argument('-l', '--lhe', action='store_true', dest='isLHErequest', + help='Check dataset when modifying requests. Fail and do not modify name if they conflict. Use for updating GS requests chained to wmLHE and pLHE requests.') args_ = parser.parse_args() return args_ @@ -42,15 +50,16 @@ def getArguments(): def checkFile(file_): # Check that CSV file exists if not os.path.isfile(file_): - print "Error: File %s does not exist." % file_ - print "Exiting with status 1." + print "Error: File {0} does not exist.".format(file_) sys.exit(1) def checkPWG(pwg_): - pwg_list = ['B2G','BPH','BTW','EGM','EWK','EXO','FSQ','FWD','HCA','HIG','HIN','JME','L1T','MUO','QCD','SMP','SUS','TAU','TOP','TRK','TSG'] + pwg_list = ['B2G', 'BPH', 'BTW', 'EGM', 'EWK', 'EXO', 'FSQ', 'FWD', 'HCA', + 'HIG', 'HIN', 'JME', 'L1T', 'MUO', 'QCD', 'SMP', 'SUS', 'TAU', + 'TOP', 'TRK', 'TSG'] # Check that PWG is valid if pwg_ not in pwg_list: - print "Error: %s is not a recognized PWG." % pwg_ + print "Error: {0} is not a recognized PWG.".format(pwg_) if pwg_ == 'XXX': print "Change the default value for flag -p to your PWG by modifying the variable defaultPWG on line 23." sys.stdout.write("Options are:") @@ -58,100 +67,136 @@ def checkPWG(pwg_): sys.stdout.write(" ") sys.stdout.write(iPWG) sys.stdout.write("\n") - print "Exiting with status 2." sys.exit(2) -def checkNotCreate(doModify_,cloneId_): +def checkNotCreate(doModify_, cloneId_): # Check that script isn't being asked to both modify and clone a request doClone = False - if cloneId_ != "": doClone = True + if cloneId_ != "": + doClone = True if doModify_ and doClone: print "Error: cannot both --modify and --clone." - print "Exiting with status 6." sys.exit(6) return doModify_ or doClone # Return variable to use in fillFields() -def exitDuplicateField(file_in_,field_): - print "Error: File %s contains multiple instances of the field %s" % (file_in_,field_) - print "Exiting with status 3." +def exitDuplicateField(file_in_, field_): + print "Error: File {0} contains multiple instances of the field {1}".format( + file_in_, field_) sys.exit(3) -def getFields(csvfile_,file_in_): +def getFields(csvfile_, file_in_): # List of indices for each field in CSV file list = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1] header = csv.reader(csvfile_).next() for ind, field in enumerate(header): - if field in ['Dataset name','Dataset Name','Dataset','dataset']: + if field in ['Dataset name', 'Dataset Name', 'Dataset', 'dataset']: #ensure no duplicate fields - if list[0] > -1: exitDuplicateField(file_in_,"Dataset name") + if list[0] > -1: + exitDuplicateField(file_in_, "Dataset name") list[0] = ind - elif field in ['EOS','eos','Eos','MCDBID','mcdbid']: - if list[1] > -1: exitDuplicateField(file_in_,"EOS") + elif field in ['EOS', 'eos', 'Eos', 'MCDBID', 'mcdbid']: + if list[1] > -1: + exitDuplicateField(file_in_, "EOS") list[1] = ind - elif field in ['Cross section [pb]','Cross section','Cross section (pb)','Cross Section','Cross Section [pb]','Cross Section (pb)','CS','CS [pb]','CS (pb)','Xsec','Xsec [pb]','Xsec (pb)']: - if list[2] > -1: exitDuplicateField(file_in_,"Cross section") + elif field in ['Cross section [pb]', 'Cross section', + 'Cross section (pb)', 'Cross Section', + 'Cross Section [pb]', 'Cross Section (pb)', 'CS', + 'CS [pb]', 'CS (pb)', 'Xsec', 'Xsec [pb]', 'Xsec (pb)']: + if list[2] > -1: + exitDuplicateField(file_in_, "Cross section") list[2] = ind - elif field in ['Total events','Total Events','Events','events','total events','Number of Events']: - if list[3] > -1: exitDuplicateField(file_in_,"Total events") + elif field in ['Total events', 'Total Events', 'Events', 'events', + 'total events', 'Number of Events']: + if list[3] > -1: + exitDuplicateField(file_in_, "Total events") list[3] = ind - elif field in ['Fragment name','Fragment Name','Generator fragment name','Generator Fragment Name','Fragment','fragment']: - if list[4] > -1: exitDuplicateField(file_in_,"Fragment name") + elif field in ['Fragment name', 'Fragment Name', + 'Generator fragment name', 'Generator Fragment Name', + 'Fragment', 'fragment']: + if list[4] > -1: + exitDuplicateField(file_in_, "Fragment name") list[4] = ind - elif field in ['Time per event [s]','Time per event','Time per event (s)','Time per Event','Time per Event [s]','Time per Event (s)','Time','Time [s]','Time (s)','time','time [s]','time (s)']: - if list[5] > -1: exitDuplicateField(file_in_,"Time per event [s]") + elif field in ['Time per event [s]', 'Time per event', + 'Time per event (s)', 'Time per Event', + 'Time per Event [s]', 'Time per Event (s)', 'Time', + 'Time [s]', 'Time (s)', 'time', 'time [s]', 'time (s)']: + if list[5] > -1: + exitDuplicateField(file_in_, "Time per event [s]") list[5] = ind - elif field in ['Size per event [kB]','Size per event','Size per event (kB)','Size per Event','Size per Event [kB]','Size per Event (kB)','size','size [kB]','size (kB)']: - if list[6] > -1: exitDuplicateField(file_in_,"Size per event [kB]") + elif field in ['Size per event [kB]', 'Size per event', + 'Size per event (kB)', 'Size per Event', + 'Size per Event [kB]', 'Size per Event (kB)', 'size', + 'size [kB]', 'size (kB)']: + if list[6] > -1: + exitDuplicateField(file_in_, "Size per event [kB]") list[6] = ind - elif field in ['Tag','tag','Fragment Tag','Fragment tag','fragment tag','sha','SHA','SHA-1','sha-1']: - if list[7] > -1: exitDuplicateField(file_in_,"Fragment tag") + elif field in ['Tag', 'tag', 'Fragment Tag', 'Fragment tag', + 'fragment tag', 'sha', 'SHA', 'SHA-1', 'sha-1']: + if list[7] > -1: + exitDuplicateField(file_in_, "Fragment tag") list[7] = ind - elif field in ['Generator','generator']: - if list[8] > -1: exitDuplicateField(file_in_,"Generator") + elif field in ['Generator', 'generator']: + if list[8] > -1: + exitDuplicateField(file_in_, "Generator") list[8] = ind - elif field in ['Filter efficiency','FilterEfficiency','filter efficiency']: - if list[9] > -1: exitDuplicateField(file_in_,"Filter efficiency") + elif field in ['Filter efficiency', 'FilterEfficiency', + 'filter efficiency']: + if list[9] > -1: + exitDuplicateField(file_in_, "Filter efficiency") list[9] = ind - elif field in ['Filter efficiency error','Filter Efficiency Error','filter efficiency error']: - if list[10] > -1: exitDuplicateField(file_in_,"Filter efficiency error") + elif field in ['Filter efficiency error', 'Filter Efficiency Error', + 'filter efficiency error']: + if list[10] > -1: + exitDuplicateField(file_in_, "Filter efficiency error") list[10] = ind - elif field in ['Match efficiency','Match Efficiency','match efficiency']: - if list[11] > -1: exitDuplicateField(file_in_,"Match efficiency") + elif field in ['Match efficiency', 'Match Efficiency', + 'match efficiency']: + if list[11] > -1: + exitDuplicateField(file_in_, "Match efficiency") list[11] = ind - elif field in ['Match efficiency error','Match Efficiency Error','match efficiency error']: - if list[12] > -1: exitDuplicateField(file_in_,"Match efficiency error") + elif field in ['Match efficiency error', 'Match Efficiency Error', + 'match efficiency error']: + if list[12] > -1: + exitDuplicateField(file_in_, "Match efficiency error") list[12] = ind - elif field in ['PWG','pwg']: - if list[13] > -1: exitDuplicateField(file_in_,"PWG") + elif field in ['PWG', 'pwg']: + if list[13] > -1: exitDuplicateField(file_in_, "PWG") list[13] = ind - elif field in ['Campaign','campaign','Member of Campaign','Member of campaign','member of campaign']: - if list[14] > -1: exitDuplicateField(file_in_,"Member of campaign") + elif field in ['Campaign', 'campaign', 'Member of Campaign', + 'Member of campaign', 'member of campaign']: + if list[14] > -1: + exitDuplicateField(file_in_, "Member of campaign") list[14] = ind - elif field in ['PrepId','PrepID','PREPID','prepid']: - if list[15] > -1: exitDuplicateField(file_in_,"PrepId") + elif field in ['PrepId', 'PrepID', 'PREPID', 'prepid']: + if list[15] > -1: + exitDuplicateField(file_in_, "PrepId") list[15] = ind - elif field in ['Sequences customise','Sequences customize']: - if list[16] > -1: exitDuplicateField(file_in_,"Sequences customise") + elif field in ['Sequences customise', 'Sequences customize']: + if list[16] > -1: + exitDuplicateField(file_in_, "Sequences customise") list[16] = ind - elif field in ['Process string','Process String']: - if list[17] > -1: exitDuplicateField(file_in_,"Process string") + elif field in ['Process string', 'Process String']: + if list[17] > -1: + exitDuplicateField(file_in_, "Process string") list[17] = ind elif field in ['Gridpack location', 'Gridpack']: - if list[18] > -1: exitDuplicateField(file_in_,"Gridpack location") + if list[18] > -1: + exitDuplicateField(file_in_, "Gridpack location") list[18] = ind elif field in ['Gridpack cards URL', 'Cards URL', 'Gridpack cards location', 'Cards location']: - if list[19] > -1: exitDuplicateField(file_in_,"Gridpack cards URL") + if list[19] > -1: + exitDuplicateField(file_in_, "Gridpack cards URL") list[19] = ind elif field in ['Notes', 'notes']: - if list[20] > -1: exitDuplicateField(file_in_,"Notes") + if list[20] > -1: + exitDuplicateField(file_in_, "Notes") list[20] = ind elif field in ['JobId', 'Local gridpack location', 'Local LHE']: continue else: - print "Error: The field %s is not valid." % field - print "Exiting with status 4." + print "Error: The field {0} is not valid.".format(field) sys.exit(4) return list @@ -175,19 +220,19 @@ def createLHEProducer(gridpack, cards): code = """import FWCore.ParameterSet.Config as cms externalLHEProducer = cms.EDProducer("ExternalLHEProducer", - args = cms.vstring('%s'), + args = cms.vstring('{0}'), nEvents = cms.untracked.uint32(5000), numberOfParameters = cms.uint32(1), outputFile = cms.string('cmsgrid_final.lhe'), scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh') -)""" % (gridpack) +)""".format(gridpack) if cards != "": code += """ # Link to cards: -# %s -""" % (cards) +# {0} +""".format(cards) return code def fillFields(csvfile, fields, campaign, PWG, notCreate_): @@ -196,7 +241,8 @@ def fillFields(csvfile, fields, campaign, PWG, notCreate_): for row in csv.reader(csvfile): num_requests += 1 tmpReq = Request() - if fields[0] > -1: tmpReq.setDataSetName(row[fields[0]]) + if fields[0] > -1: + tmpReq.setDataSetName(row[fields[0]]) if fields[1] > -1: tmpReq.setMCDBID(row[fields[1]]) elif not notCreate_: @@ -205,17 +251,23 @@ def fillFields(csvfile, fields, campaign, PWG, notCreate_): tmpReq.setCS(row[fields[2]]) elif not notCreate_: tmpReq.setCS(1.0) - if fields[3] > -1: tmpReq.setEvts(row[fields[3]]) + if fields[3] > -1: + tmpReq.setEvts(row[fields[3]]) if fields[14] > -1: campaign = row[fields[14]] tmpReq.setCamp(campaign) elif campaign is not None: tmpReq.setCamp(campaign) - if fields[4] > -1: tmpReq.setFrag(formatFragment(row[fields[4]],campaign)) - if fields[5] > -1: tmpReq.setTime(row[fields[5]]) - if fields[6] > -1: tmpReq.setSize(row[fields[6]]) - if fields[7] > -1: tmpReq.setTag(row[fields[7]]) - if fields[8] > -1: tmpReq.setGen(row[fields[8]].split(" ")) # Multiple generators separated by spaces + if fields[4] > -1: + tmpReq.setFrag(formatFragment(row[fields[4]],campaign)) + if fields[5] > -1: + tmpReq.setTime(row[fields[5]]) + if fields[6] > -1: + tmpReq.setSize(row[fields[6]]) + if fields[7] > -1: + tmpReq.setTag(row[fields[7]]) + if fields[8] > -1: + tmpReq.setGen(row[fields[8]].split(" ")) # Multiple generators separated by spaces if fields[9] > -1: tmpReq.setFiltEff(row[fields[9]]) elif not notCreate_: @@ -255,70 +307,94 @@ def fillFields(csvfile, fields, campaign, PWG, notCreate_): def createRequests(requests, num_requests, doDryRun, useDev): # Create new requests based on campaign and PWG - mcm = restful( dev=useDev ) # Get McM connection + mcm = restful(dev=useDev) # Get McM connection if not doDryRun: - print "Adding %d requests to McM." % num_requests + print "Adding {0} requests to McM.".format(num_requests) else: - print "Dry run. %d requests will not be added to McM." % num_requests + print "Dry run. {0} requests will not be added to McM.".format( + num_requests) for reqFields in requests: if not reqFields.useCamp(): print "A campaign is needed for new requests." continue # Create new request's dictionary - new_req = {'pwg':reqFields.getPWG(),'member_of_campaign':reqFields.getCamp(),'mcdb_id':reqFields.getMCDBID()} + new_req = {'pwg': reqFields.getPWG(), + 'member_of_campaign': reqFields.getCamp(), + 'mcdb_id': reqFields.getMCDBID()} # Fill dictionary with fields - if reqFields.useDataSetName(): new_req['dataset_name'] = reqFields.getDataSetName() - if reqFields.useEvts(): new_req['total_events'] = reqFields.getEvts() - if reqFields.useFrag(): new_req['name_of_fragment'] = reqFields.getFrag() - if reqFields.useTag(): new_req['fragment_tag'] = reqFields.getTag() - if reqFields.useMcMFrag(): new_req['fragment'] = reqFields.getMcMFrag() - if reqFields.useTime(): new_req['time_event'] = reqFields.getTime() - if reqFields.useSize(): new_req['size_event'] = reqFields.getSize() - if reqFields.useGen(): new_req['generators'] = reqFields.getGen() + if reqFields.useDataSetName(): + new_req['dataset_name'] = reqFields.getDataSetName() + if reqFields.useEvts(): + new_req['total_events'] = reqFields.getEvts() + if reqFields.useFrag(): + new_req['name_of_fragment'] = reqFields.getFrag() + if reqFields.useTag(): + new_req['fragment_tag'] = reqFields.getTag() + if reqFields.useMcMFrag(): + new_req['fragment'] = reqFields.getMcMFrag() + if reqFields.useTime(): + new_req['time_event'] = reqFields.getTime() + if reqFields.useSize(): + new_req['size_event'] = reqFields.getSize() + if reqFields.useGen(): + new_req['generators'] = reqFields.getGen() # Sequences might need to be added below with generator parameters - if reqFields.useSequencesCustomise(): new_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() - if reqFields.useProcessString(): new_req['process_string'] = reqFields.getProcessString() - if reqFields.useNotes(): new_req['notes'] = reqFields.getNotes() + if reqFields.useSequencesCustomise(): + new_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() + if reqFields.useProcessString(): + new_req['process_string'] = reqFields.getProcessString() + if reqFields.useNotes(): + new_req['notes'] = reqFields.getNotes() if not doDryRun: answer = mcm.putA('requests', new_req) # Create request if answer['results']: # Cannot fill generator parameters while creating a new request # Modify newly created request with generator parameters - mod_req = mcm.getA('requests',answer['prepid']) # Get newly created request + # Get newly created request + mod_req = mcm.getA('requests', answer['prepid']) # Fill generator parameters - mod_req['generator_parameters'][0]['cross_section'] = reqFields.getCS() - mod_req['generator_parameters'][0]['filter_efficiency'] = reqFields.getFiltEff() - mod_req['generator_parameters'][0]['filter_efficiency_error'] = reqFields.getFiltEffErr() - mod_req['generator_parameters'][0]['match_efficiency'] = reqFields.getMatchEff() - mod_req['generator_parameters'][0]['match_efficiency_error'] = reqFields.getMatchEffErr() - update_answer = mcm.updateA('requests',mod_req) # Update request with generator parameters + mod_req['generator_parameters'][0]['cross_section'] \ + = reqFields.getCS() + mod_req['generator_parameters'][0]['filter_efficiency'] \ + = reqFields.getFiltEff() + mod_req['generator_parameters'][0]['filter_efficiency_error'] \ + = reqFields.getFiltEffErr() + mod_req['generator_parameters'][0]['match_efficiency'] \ + = reqFields.getMatchEff() + mod_req['generator_parameters'][0]['match_efficiency_error'] \ + = reqFields.getMatchEffErr() + # Update request with generator parameters + update_answer = mcm.updateA('requests', mod_req) if update_answer['results']: - print answer['prepid'],"created" + print "{0} created".format(answer['prepid']) else: - print answer['prepid'],"created but generator parameters not set" + print "{0} created but generator parameters not set".format( + answer['prepid']) else: if reqFields.useDataSetname(): - print reqFields.getDataSetName(),"failed to be created" + print "{0} failed to be created".format( + reqFields.getDataSetName()) else: print "A request has failed to be created" else: if reqFields.useDataSetName(): - print reqFields.getDataSetName(),"not created" + print "{0} not created".format(reqFields.getDataSetName()) else: print "request not created" pprint.pprint(new_req) def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): # Modify existing request based on PrepId - mcm = restful( dev=useDev ) # Get McM connection + mcm = restful(dev=useDev) # Get McM connection if not doDryRun: - print "Modifying %d requests to McM." % num_requests + print "Modifying {0} requests to McM.".format(num_requests) else: - print "Dry run. %d requests will not be modified in McM." % num_requests + print "Dry run. {0} requests will not be modified in McM.".format( + num_requests) for reqFields in requests: # Get request from McM if isLHErequest: @@ -326,11 +402,11 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): print "\033[1;31mDataset name missing\033[1;m" continue elif not reqFields.useCamp(): - print "\033[1;31m%s modification failed. Must provide campaign.\033[1;m" \ - % (reqFields.getDataSetName()) + print "\033[1;31m{0} modification failed. Must provide campaign.\033[1;m".format( + reqFields.getDataSetName()) continue - query_string = "dataset_name=%s&member_of_campaign=%s" \ - % (reqFields.getDataSetName(), reqFields.getCamp()) + query_string = "dataset_name={0}&member_of_campaign={1}".format( + reqFields.getDataSetName(), reqFields.getCamp()) failed_to_get = True for tries in range(3): time.sleep(0.1) @@ -339,8 +415,8 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): failed_to_get = False break if failed_to_get: - print "\033[1;31m%s modification failed. Could not get request from McM.\033[1;m"\ - % (reqFields.getDataSetName()) + print "\033[1;31m{0} modification failed. Could not get request from McM.\033[1;m".format( + reqFields.getDataSetName()) continue if len(mod_req_list) > 1: print "\033[1;31m{0} modification failed. Too many requests match query.\033[1;m".format( @@ -356,91 +432,149 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): print "\033[1;31mPrepId is missing.\033[1;m" continue time.sleep(0.1) - mod_req = mcm.getA('requests',reqFields.getPrepId()) + mod_req = mcm.getA('requests', reqFields.getPrepId()) if reqFields.useDataSetName() and not isLHErequest: mod_req['dataset_name'] = reqFields.getDataSetName() - if reqFields.useMCDBID(): mod_req['mcdb_id'] = reqFields.getMCDBID() - if reqFields.useEvts(): mod_req['total_events'] = reqFields.getEvts() - if reqFields.useFrag(): mod_req['name_of_fragment'] = reqFields.getFrag() - if reqFields.useTag(): mod_req['fragment_tag'] = reqFields.getTag() - if reqFields.useMcMFrag(): mod_req['fragment'] = reqFields.getMcMFrag() - if reqFields.useTime(): mod_req['time_event'] = reqFields.getTime() - if reqFields.useSize(): mod_req['size_event'] = reqFields.getSize() - if reqFields.useGen(): mod_req['generators'] = reqFields.getGen() - if (reqFields.useCS() or reqFields.useFiltEff() or reqFields.useFiltEffErr() or reqFields.useMatchEff() or reqFields.useMatchEffErr()) and mod_req['generator_parameters'] == []: - mod_req['generator_parameters'] = [{'match_efficiency_error': 0.0, 'match_efficiency': 1.0, 'filter_efficiency': 1.0, 'version': 0, 'cross_section': 1.0, 'filter_efficiency_error': 0.0}] - if reqFields.useCS(): mod_req['generator_parameters'][0]['cross_section'] = reqFields.getCS() - if reqFields.useFiltEff(): mod_req['generator_parameters'][0]['filter_efficiency'] = reqFields.getFiltEff() - if reqFields.useFiltEffErr(): mod_req['generator_parameters'][0]['filter_efficiency_error'] = reqFields.getFiltEffErr() - if reqFields.useMatchEff(): mod_req['generator_parameters'][0]['match_efficiency'] = reqFields.getMatchEff() - if reqFields.useMatchEffErr(): mod_req['generator_parameters'][0]['match_efficiency_error'] = reqFields.getMatchEffErr() - if reqFields.useSequencesCustomise(): mod_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() - if reqFields.useProcessString(): mod_req['process_string'] = reqFields.getProcessString() - if reqFields.useNotes(): mod_req['notes'] = reqFields.getNotes() + if reqFields.useMCDBID(): + mod_req['mcdb_id'] = reqFields.getMCDBID() + if reqFields.useEvts(): + mod_req['total_events'] = reqFields.getEvts() + if reqFields.useFrag(): + mod_req['name_of_fragment'] = reqFields.getFrag() + if reqFields.useTag(): + mod_req['fragment_tag'] = reqFields.getTag() + if reqFields.useMcMFrag(): + mod_req['fragment'] = reqFields.getMcMFrag() + if reqFields.useTime(): + mod_req['time_event'] = reqFields.getTime() + if reqFields.useSize(): + mod_req['size_event'] = reqFields.getSize() + if reqFields.useGen(): + mod_req['generators'] = reqFields.getGen() + if (reqFields.useCS() or reqFields.useFiltEff() + or reqFields.useFiltEffErr() or reqFields.useMatchEff() + or reqFields.useMatchEffErr()) and mod_req['generator_parameters'] == []: + mod_req['generator_parameters'] = [{'match_efficiency_error': 0.0, + 'match_efficiency': 1.0, + 'filter_efficiency': 1.0, + 'version': 0, + 'cross_section': 1.0, + 'filter_efficiency_error': 0.0}] + if reqFields.useCS(): + mod_req['generator_parameters'][0]['cross_section'] \ + = reqFields.getCS() + if reqFields.useFiltEff(): + mod_req['generator_parameters'][0]['filter_efficiency'] \ + = reqFields.getFiltEff() + if reqFields.useFiltEffErr(): + mod_req['generator_parameters'][0]['filter_efficiency_error'] \ + = reqFields.getFiltEffErr() + if reqFields.useMatchEff(): + mod_req['generator_parameters'][0]['match_efficiency'] \ + = reqFields.getMatchEff() + if reqFields.useMatchEffErr(): + mod_req['generator_parameters'][0]['match_efficiency_error'] \ + = reqFields.getMatchEffErr() + if reqFields.useSequencesCustomise(): + mod_req['sequences'][0]['customise'] \ + = reqFields.getSequencesCustomise() + if reqFields.useProcessString(): + mod_req['process_string'] = reqFields.getProcessString() + if reqFields.useNotes(): + mod_req['notes'] = reqFields.getNotes() if not doDryRun: - answer = mcm.updateA('requests',mod_req) # Update request + answer = mcm.updateA('requests', mod_req) # Update request if answer['results']: if not isLHErequest: - print "\033[1;32m%s modified\033[1;m" % (reqFields.getPrepId()) + print "\033[1;32m{0} modified\033[1;m".format( + reqFields.getPrepId()) else: - print "\033[1;32m%s in %s modified\033[1;m" % (reqFields.getDataSetName(), - reqFields.getCamp()) + print "\033[1;32m{0} in {1} modified\033[1;m".format( + reqFields.getDataSetName(), reqFields.getCamp()) else: if not isLHErequest: - print "\033[1;31m",reqFields.getPrepId(),"failed to be modified\033[1;m" + print "\033[1;31m{0} failed to be modified\033[1;m".format( + reqFields.getPrepId()) else: - print "\033[1;31m",reqFields.getDataSetName(),"failed to be modified\033[1;m" + print "\033[1;31m{0} failed to be modified\033[1;m".format( + reqFields.getDataSetName()) else: if not isLHErequest: - print reqFields.getPrepId(),"not modified" + print "{0} not modified".format(reqFields.getPrepId()) pprint.pprint(mod_req) else: - print reqFields.getDataSetName(),"not modified" + print "{0} not modified".format(reqFields.getDataSetName()) pprint.pprint(mod_req) def cloneRequests(requests, num_requests, doDryRun, useDev, cloneId_): # Create new requests be cloning an old one based on PrepId - mcm = restful( dev=useDev ) # Get McM connection + mcm = restful(dev=useDev) # Get McM connection if not doDryRun: - print "Adding %d requests to McM using clone." % num_requests + print "Adding {0} requests to McM using clone.".format(num_requests) else: - print "Dry run. %d requests will not be added to McM using clone." % num_requests + print "Dry run. {0} requests will not be added to McM using clone.".format( + num_requests) for reqFields in requests: - clone_req = mcm.getA('requests',cloneId_) # Get request to clone - if reqFields.useDataSetName(): clone_req['dataset_name'] = reqFields.getDataSetName() - if reqFields.useMCDBID(): clone_req['mcdb_id'] = reqFields.getMCDBID() - if reqFields.useEvts(): clone_req['total_events'] = reqFields.getEvts() - if reqFields.useFrag(): clone_req['name_of_fragment'] = reqFields.getFrag() - if reqFields.useTag(): clone_req['fragment_tag'] = reqFields.getTag() - if reqFields.useMcMFrag(): clone_req['fragment'] = reqFields.getMcMFrag() - if reqFields.useTime(): clone_req['time_event'] = reqFields.getTime() - if reqFields.useSize(): clone_req['size_event'] = reqFields.getSize() - if reqFields.useGen(): clone_req['generators'] = reqFields.getGen() - if reqFields.useCS(): clone_req['generator_parameters'][0]['cross_section'] = reqFields.getCS() - if reqFields.useFiltEff(): clone_req['generator_parameters'][0]['filter_efficiency'] = reqFields.getFiltEff() - if reqFields.useFiltEffErr(): clone_req['generator_parameters'][0]['filter_efficiency_error'] = reqFields.getFiltEffErr() - if reqFields.useMatchEff(): clone_req['generator_parameters'][0]['match_efficiency'] = reqFields.getMatchEff() - if reqFields.useMatchEffErr(): clone_req['generator_parameters'][0]['match_efficiency_error'] = reqFields.getMatchEffErr() - if reqFields.useSequencesCustomise(): clone_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() - if reqFields.useProcessString(): clone_req['process_string'] = reqFields.getProcessString() - if reqFields.useNotes(): clone_req['notes'] = reqFields.getNotes() + clone_req = mcm.getA('requests', cloneId_) # Get request to clone + if reqFields.useDataSetName(): + clone_req['dataset_name'] = reqFields.getDataSetName() + if reqFields.useMCDBID(): + clone_req['mcdb_id'] = reqFields.getMCDBID() + if reqFields.useEvts(): + clone_req['total_events'] = reqFields.getEvts() + if reqFields.useFrag(): + clone_req['name_of_fragment'] = reqFields.getFrag() + if reqFields.useTag(): + clone_req['fragment_tag'] = reqFields.getTag() + if reqFields.useMcMFrag(): + clone_req['fragment'] = reqFields.getMcMFrag() + if reqFields.useTime(): + clone_req['time_event'] = reqFields.getTime() + if reqFields.useSize(): + clone_req['size_event'] = reqFields.getSize() + if reqFields.useGen(): + clone_req['generators'] = reqFields.getGen() + if reqFields.useCS(): + clone_req['generator_parameters'][0]['cross_section'] \ + = reqFields.getCS() + if reqFields.useFiltEff(): + clone_req['generator_parameters'][0]['filter_efficiency'] \ + = reqFields.getFiltEff() + if reqFields.useFiltEffErr(): + clone_req['generator_parameters'][0]['filter_efficiency_error'] \ + = reqFields.getFiltEffErr() + if reqFields.useMatchEff(): + clone_req['generator_parameters'][0]['match_efficiency'] \ + = reqFields.getMatchEff() + if reqFields.useMatchEffErr(): + clone_req['generator_parameters'][0]['match_efficiency_error'] \ + = reqFields.getMatchEffErr() + if reqFields.useSequencesCustomise(): + clone_req['sequences'][0]['customise'] \ + = reqFields.getSequencesCustomise() + if reqFields.useProcessString(): + clone_req['process_string'] = reqFields.getProcessString() + if reqFields.useNotes(): + clone_req['notes'] = reqFields.getNotes() if not doDryRun: - answer = mcm.clone(cloneId_,clone_req) # Clone request + answer = mcm.clone(cloneId_, clone_req) # Clone request if answer['results']: - print answer['prepid'],"created using clone" + print "{0} created using clone".format(answer['prepid']) else: if reqFields.useDataSetName(): - print reqFields.getDataSetName(),"failed to be created using clone" + print "{0} failed to be created using clone".format( + reqFields.getDataSetName()) else: print "request failed to be created using clone" else: if reqFields.useDataSetName(): - print reqFields.getDataSetName(),"not created using clone" + print "{0} not created using clone".format( + reqFields.getDataSetName()) else: print "request not created using clone" pprint.pprint(clone_req) @@ -450,23 +584,26 @@ def main(): checkPWG(args.pwg) # Make sure PWG is an actual PWG # Check that script is not asked to both modify and clone # Store variable that is true if script is asked to modify or clone - notCreate = checkNotCreate(args.doModify,args.cloneId) + notCreate = checkNotCreate(args.doModify, args.cloneId) checkFile(args.file_in) # Ensure CSV file exists if args.useDev: print "Using dev/test instance." - csvfile = open(args.file_in,'r') # Open CSV file - fields = getFields(csvfile,args.file_in) # Get list of field indices + csvfile = open(args.file_in, 'r') # Open CSV file + fields = getFields(csvfile, args.file_in) # Get list of field indices # Fill list of request objects with fields from CSV and get number of requests - requests, num_requests = fillFields(csvfile, fields, args.campaign, args.pwg, notCreate) + requests, num_requests = fillFields(csvfile, fields, args.campaign, + args.pwg, notCreate) if args.doModify: # Modify existing requests - modifyRequests(requests, num_requests, args.doDryRun, args.useDev, args.isLHErequest) + modifyRequests(requests, num_requests, args.doDryRun, args.useDev, + args.isLHErequest) elif args.cloneId != "": # Create new requests using clone - cloneRequests(requests, num_requests, args.doDryRun, args.useDev, args.cloneId) + cloneRequests(requests, num_requests, args.doDryRun, args.useDev, + args.cloneId) else: # Create new requests createRequests(requests, num_requests, args.doDryRun, args.useDev) diff --git a/testRequests.py b/testRequests.py index 90ceb6e..bac1fa1 100755 --- a/testRequests.py +++ b/testRequests.py @@ -32,8 +32,6 @@ def getArguments(): parser.add_argument('-o', '--output', dest='output', default='test.csv', help='Output CSV file') parser.add_argument('-n', dest='nEvents', help='Number of events to test.') - parser.add_argument('-v', '--version', action='version', - version='%(prog)s v1.0') args_ = parser.parse_args() return args_ @@ -43,14 +41,13 @@ def fillIDRange(pwg, campaign, first, last): last = int(last) requests = [] if first > last: - print "Error: PrepID range out of order. %s-%s-%05d > %s-%s-%05d" % ( - pwg, campaign, first, pwg, campaign, last) - print "Exiting with status 4." + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) sys.exit(4) for number in range(first, last+1): tmpReq = Request() - tmpReq.setPrepId("%s-%s-%05d" % (pwg, campaign, number)) + tmpReq.setPrepId("{0}-{1}-{2:05d}".format(pwg, campaign, number)) requests.append(tmpReq) return requests @@ -69,11 +66,9 @@ def parseIDList(compactList): elif len(splitSubList) == 6: if splitSubList[0] != splitSubList[3]: print "Error: PrepID range must be for the same PWG." - print "Exiting with status 4" sys.exit(4) if splitSubList[1] != splitSubList[4]: print "Error: PrepID range must be for the same campaign." - print "Exiting with status 4" sys.exit(4) requests = requests + fillIDRange(splitSubList[0], splitSubList[1], splitSubList[2], splitSubList[5]) @@ -91,42 +86,42 @@ def getTestScript(PrepID, nEvents): get_test = "" if nEvents is None: get_test = "curl -s --insecure \ -https://cms-pdmv.cern.ch/mcm/public/restapi/%s/get_test/%s -o %s.sh" % ( +https://cms-pdmv.cern.ch/mcm/public/restapi/{0}/get_test/{1} -o {2}.sh".format( request_type, PrepID, PrepID) else: + # add "/N" to end of URL to get N events get_test = "curl -s --insecure \ -https://cms-pdmv.cern.ch/mcm/public/restapi/%s/get_test/%s/%s -o %s.sh" % ( +https://cms-pdmv.cern.ch/mcm/public/restapi/{0}/get_test/{1}/{2} -o {3}.sh".format( request_type, PrepID, nEvents, PrepID) - # add "/N" to end of URL to get N events print get_test subprocess.call(get_test, shell=True) if request_type == "chained_requests" and nEvents is not None: - filename = "%s.sh" % (PrepID) - tmpfilename = "tmp%s.sh" % (PrepID) + filename = "{0}.sh".format(PrepID) + tmpfilename = "tmp{0}.sh".format(PrepID) inputfile = open(filename, 'r') outputfile = open(tmpfilename, 'w') for line in inputfile: - outline = re.sub('(.*--eventcontent LHE.*-n) \d*( .*)', r'\1 %s\2' - % (nEvents), line) - outline = re.sub('(.*--eventcontent DQM.*-n) \d*( .*)', r'\1 %s\2' - % (nEvents), outline) - outline = re.sub('(.*--eventcontent RAWSIM.*-n) \d*( .*)', r'\1 %s\2' - % (nEvents), outline) + outline = re.sub('(.*--eventcontent LHE.*-n) \d*( .*)', + r'\1 {0}\2'.format(nEvents), line) + outline = re.sub('(.*--eventcontent DQM.*-n) \d*( .*)', + r'\1 {0}\2'.format(nEvents), outline) + outline = re.sub('(.*--eventcontent RAWSIM.*-n) \d*( .*)', + r'\1 {0}\2'.format(nEvents), outline) outputfile.write(outline) inputfile.close() outputfile.close() os.rename(tmpfilename, filename) - subprocess.call("chmod 755 %s.sh" % (PrepID), shell=True) + subprocess.call("chmod 755 {0}.sh".format(PrepID), shell=True) return def submitToBatch(PrepId): - batch_command = "bsub -q 8nh %s.sh" % (PrepId) + batch_command = "bsub -q 8nh {0}.sh".format(PrepId) print batch_command output = subprocess.Popen(batch_command, stdout=subprocess.PIPE, shell=True).communicate()[0] - match = re.match('Job <(\d*)> is',output) + match = re.match('Job <(\d*)> is', output) jobID = match.group(1) return jobID @@ -137,7 +132,7 @@ def createTest(compactPrepIDList, outputFile, nEvents): csvfile.writerow(['PrepId', 'JobId', 'Time per event [s]', 'Size per event [kB]']) - print "Testing %d requests" % (len(requests)) + print "Testing {0} requests".format(len(requests)) for req in requests: getTestScript(req.getPrepId(), nEvents) jobID = submitToBatch(req.getPrepId()) @@ -147,7 +142,7 @@ def createTest(compactPrepIDList, outputFile, nEvents): csvfile.writerow([req.getPrepId(), req.getJobID(), "", ""]) else: mcm = restful(dev=False) # Get McM connection - mcm_req = mcm.getA('chained_requests',req.getPrepId()) + mcm_req = mcm.getA('chained_requests', req.getPrepId()) wmLHEPrepId = mcm_req['chain'][0] GSPrepId = mcm_req['chain'][1] csvfile.writerow([wmLHEPrepId, req.getJobID(), "", ""]) @@ -155,8 +150,8 @@ def createTest(compactPrepIDList, outputFile, nEvents): return def exitDuplicateField(file_in_,field_): - print "Error: File %s contains multiple instances of the field %s" % (file_in_,field_) - print "Exiting with status 5." + print "Error: File {0} contains multiple instances of the field {1}".format( + file_in_, field_) sys.exit(5) def getFields(csvfile): @@ -167,92 +162,112 @@ def getFields(csvfile): for ind, field in enumerate(header): if field in ['Dataset name', 'Dataset Name', 'Dataset', 'dataset']: #ensure no duplicate fields - if list[0] > -1: exitDuplicateField(file_in_, "Dataset name") + if list[0] > -1: + exitDuplicateField(file_in_, "Dataset name") list[0] = ind elif field in ['EOS', 'eos', 'Eos', 'MCDBID', 'mcdbid']: - if list[1] > -1: exitDuplicateField(file_in_, "EOS") + if list[1] > -1: + exitDuplicateField(file_in_, "EOS") list[1] = ind elif field in ['Cross section [pb]', 'Cross section', 'Cross section (pb)', 'Cross Section', 'Cross Section [pb]', 'Cross Section (pb)', 'CS', 'CS [pb]', 'CS (pb)', 'Xsec', 'Xsec [pb]', 'Xsec (pb)']: - if list[2] > -1: exitDuplicateField(file_in_, "Cross section") + if list[2] > -1: + exitDuplicateField(file_in_, "Cross section") list[2] = ind elif field in ['Total events', 'Total Events', 'Events', 'events', 'total events', 'Number of Events']: - if list[3] > -1: exitDuplicateField(file_in_, "Total events") + if list[3] > -1: + exitDuplicateField(file_in_, "Total events") list[3] = ind elif field in ['Fragment name', 'Fragment Name', 'Generator fragment name', 'Generator Fragment Name', 'Fragment', 'fragment']: - if list[4] > -1: exitDuplicateField(file_in_, "Fragment name") + if list[4] > -1: + exitDuplicateField(file_in_, "Fragment name") list[4] = ind elif field in ['Time per event [s]', 'Time per event', 'Time per event (s)', 'Time per Event', 'Time per Event [s]', 'Time per Event (s)', 'Time', 'Time [s]', 'Time (s)', 'time', 'time [s]', 'time (s)']: - if list[5] > -1: exitDuplicateField(file_in_, "Time per event [s]") + if list[5] > -1: + exitDuplicateField(file_in_, "Time per event [s]") list[5] = ind elif field in ['Size per event [kB]', 'Size per event', 'Size per event (kB)', 'Size per Event', 'Size per Event [kB]', 'Size per Event (kB)', 'size', 'size [kB]', 'size (kB)']: - if list[6] > -1: exitDuplicateField(file_in_, "Size per event [kB]") + if list[6] > -1: + exitDuplicateField(file_in_, "Size per event [kB]") list[6] = ind elif field in ['Tag', 'tag', 'Fragment Tag', 'Fragment tag', 'fragment tag', 'sha', 'SHA', 'SHA-1', 'sha-1']: - if list[7] > -1: exitDuplicateField(file_in_, "Fragment tag") + if list[7] > -1: + exitDuplicateField(file_in_, "Fragment tag") list[7] = ind elif field in ['Generator', 'generator']: - if list[8] > -1: exitDuplicateField(file_in_, "Generator") + if list[8] > -1: + exitDuplicateField(file_in_, "Generator") list[8] = ind elif field in ['Filter efficiency', 'FilterEfficiency', 'filter efficiency']: - if list[9] > -1: exitDuplicateField(file_in_, "Filter efficiency") + if list[9] > -1: + exitDuplicateField(file_in_, "Filter efficiency") list[9] = ind elif field in ['Filter efficiency error', 'Filter Efficiency Error', 'filter efficiency error']: - if list[10] > -1: exitDuplicateField(file_in_, - "Filter efficiency error") + if list[10] > -1: + exitDuplicateField(file_in_, "Filter efficiency error") list[10] = ind elif field in ['Match efficiency', 'Match Efficiency', 'match efficiency']: - if list[11] > -1: exitDuplicateField(file_in_, "Match efficiency") + if list[11] > -1: + exitDuplicateField(file_in_, "Match efficiency") list[11] = ind elif field in ['Match efficiency error', 'Match Efficiency Error', 'match efficiency error']: - if list[12] > -1: exitDuplicateField(file_in_, - "Match efficiency error") + if list[12] > -1: + exitDuplicateField(file_in_, "Match efficiency error") list[12] = ind - elif field in ['PWG','pwg']: - if list[13] > -1: exitDuplicateField(file_in_, "PWG") + elif field in ['PWG', 'pwg']: + if list[13] > -1: + exitDuplicateField(file_in_, "PWG") list[13] = ind elif field in ['Campaign', 'campaign', 'Member of Campaign', 'Member of campaign', 'member of campaign']: - if list[14] > -1: exitDuplicateField(file_in_, "Member of campaign") + if list[14] > -1: + exitDuplicateField(file_in_, "Member of campaign") list[14] = ind elif field in ['PrepId', 'PrepID', 'PREPID', 'prepid']: - if list[15] > -1: exitDuplicateField(file_in_, "PrepId") + if list[15] > -1: + exitDuplicateField(file_in_, "PrepId") list[15] = ind elif field in ['Sequences customise', 'Sequences customize']: - if list[16] > -1: exitDuplicateField(file_in_, "Sequences customise") + if list[16] > -1: + exitDuplicateField(file_in_, "Sequences customise") list[16] = ind elif field in ['Process string', 'Process String']: - if list[17] > -1: exitDuplicateField(file_in_, "Process string") + if list[17] > -1: + exitDuplicateField(file_in_, "Process string") list[17] = ind elif field in ['Gridpack location', 'Gridpack']: - if list[18] > -1: exitDuplicateField(file_in_, "Gridpack location") + if list[18] > -1: + exitDuplicateField(file_in_, "Gridpack location") list[18] = ind elif field in ['Gridpack cards URL', 'Cards URL', 'Gridpack cards location', 'Cards location']: - if list[19] > -1: exitDuplicateField(file_in_, "Gridpack cards URL") + if list[19] > -1: + exitDuplicateField(file_in_, "Gridpack cards URL") list[19] = ind elif field in ['JobId']: - if list[20] > -1: exitDuplicateField(file_in_, "JobId") + if list[20] > -1: + exitDuplicateField(file_in_, "JobId") list[20] = ind + elif field in ['Local gridpack location', 'Local LHE']: + continue else: - print "Error: The field %s is not valid." % (field) - print "Exiting with status 5." + print "Error: The field {0} is not valid.".format(field) sys.exit(6) return list @@ -263,22 +278,27 @@ def fillFields(csvfile, fields): for row in csv.reader(csvfile): num_requests += 1 tmpReq = Request() - if fields[0] > -1: tmpReq.setDataSetName(row[fields[0]]) + if fields[0] > -1: + tmpReq.setDataSetName(row[fields[0]]) if fields[1] > -1: tmpReq.setMCDBID(row[fields[1]]) if fields[2] > -1: tmpReq.setCS(row[fields[2]]) - if fields[3] > -1: tmpReq.setEvts(row[fields[3]]) + if fields[3] > -1: + tmpReq.setEvts(row[fields[3]]) if fields[14] > -1: campaign = row[fields[14]] tmpReq.setCamp(campaign) - if fields[4] > -1: tmpReq.setFrag(formatFragment(row[fields[4]],campaign)) + if fields[4] > -1: + tmpReq.setFrag(formatFragment(row[fields[4]],campaign)) if fields[5] > -1 and row[fields[5]] != "": tmpReq.setTime(row[fields[5]]) if fields[6] > -1 and row[fields[6]] != "": tmpReq.setSize(row[fields[6]]) - if fields[7] > -1: tmpReq.setTag(row[fields[7]]) - if fields[8] > -1: tmpReq.setGen(row[fields[8]].split(" ")) # Multiple generators separated by spaces + if fields[7] > -1: + tmpReq.setTag(row[fields[7]]) + if fields[8] > -1: + tmpReq.setGen(row[fields[8]].split(" ")) # Multiple generators separated by spaces if fields[9] > -1: tmpReq.setFiltEff(row[fields[9]]) if fields[10] > -1: @@ -331,7 +351,8 @@ def getTimeSizeFromFile(stdoutFile, iswmLHE): if match is not None: nEvents = float(match.group(1)) continue - match = re.match(' ', line) + match = re.match(' ', + line) if match is not None: totalSize = float(match.group(1)) continue @@ -352,7 +373,7 @@ def getTimeSize(requests): number_complete = 0 for req in requests: if not req.useTime() or not req.useSize(): - stdoutFile = "LSFJOB_%s/STDOUT" % (req.getJobID()) + stdoutFile = "LSFJOB_{0}/STDOUT".format(req.getJobID()) if os.path.exists(stdoutFile): number_complete += 1 iswmLHE = False @@ -367,9 +388,9 @@ def getTimeSize(requests): number_complete += 1 if number_complete == len(requests): - print "Extracted info for all %d requests." % (len(requests)) + print "Extracted info for all {0} requests.".format(len(requests)) else: - print "Extracted info for %d of %d requests. %d requests remain." % ( + print "Extracted info for {0} of {1} requests. {2} requests remain.".format( number_complete, len(requests), len(requests) - number_complete) return @@ -382,7 +403,7 @@ def extractTest(csvFile): getTimeSize(requests) - csvfile = open(csvFile,'w') + csvfile = open(csvFile, 'w') rewriteCSVFile(csvfile, requests) return @@ -393,7 +414,7 @@ def main(): print "Error: Cannot use both -i and -f." sys.exit(1) elif args.ids: - createTest(args.ids,args.output,args.nEvents) + createTest(args.ids, args.output, args.nEvents) elif args.csv: extractTest(args.csv) else: diff --git a/validateChains.py b/validateChains.py index 52631e6..491dae4 100755 --- a/validateChains.py +++ b/validateChains.py @@ -34,8 +34,8 @@ def fillIDRange(pwg, campaign, first, last): last = int(last) chains = [] if first > last: - print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {3}-{4}-{5:05d}".format( - pwg, campaign, first, pwg, campaign, last) + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) sys.exit(1) for number in range(first, last+1): @@ -72,21 +72,22 @@ def parseIDList(compactList): def validate(chains): mcm = restful(dev=False) - print "Validating %d chained requests" % (len(chains)) + print "Validating {0} chained requests".format(len(chains)) for PrepID in chains: - url = 'restapi/chained_requests/test/%s' % (PrepID) + url = 'restapi/chained_requests/test/{0}'.format(PrepID) chain_output = mcm.get(url) if chain_output['results']: - print "%s validating" % (PrepID) + print "{0} validating".format(PrepID) else: - print "%s will not be validated, due to the following reason: \n %s" % (PrepID,chain_output['message']) + print "{0} will not be validated, due to the following reason:\n{1}".format( + PrepID, chain_output['message']) def main(): - args = getArguments() # Setup flags and get arguments - chains = parseIDList(args.ids) # Get list of chains and check args - validate(chains) # Tell McM to validate chains + args = getArguments() # Setup flags and get arguments + chains = parseIDList(args.ids) # Get list of chains and check args + validate(chains) # Tell McM to validate chains if __name__ == '__main__': From e2e0eea83e505b9c896e9a78d7c7ec84b2d1a0ea Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 21 Oct 2015 22:21:16 +0200 Subject: [PATCH 02/64] Ignore column with LHE --- manageRequests.py | 2 +- testRequests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manageRequests.py b/manageRequests.py index 17f3d83..59986b3 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -193,7 +193,7 @@ def getFields(csvfile_, file_in_): if list[20] > -1: exitDuplicateField(file_in_, "Notes") list[20] = ind - elif field in ['JobId', 'Local gridpack location', 'Local LHE']: + elif field in ['JobId', 'Local gridpack location', 'Local LHE', 'LHE']: continue else: print "Error: The field {0} is not valid.".format(field) diff --git a/testRequests.py b/testRequests.py index bac1fa1..2827597 100755 --- a/testRequests.py +++ b/testRequests.py @@ -264,7 +264,7 @@ def getFields(csvfile): if list[20] > -1: exitDuplicateField(file_in_, "JobId") list[20] = ind - elif field in ['Local gridpack location', 'Local LHE']: + elif field in ['Local gridpack location', 'Local LHE', 'LHE']: continue else: print "Error: The field {0} is not valid.".format(field) From a98e3b3d9cc21c3546c52d09a3c9adc167f30ae2 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 22 Oct 2015 01:11:28 +0200 Subject: [PATCH 03/64] Color remaining output and don't make text bold --- manageRequests.py | 50 +++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/manageRequests.py b/manageRequests.py index 59986b3..ce1990e 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -369,21 +369,22 @@ def createRequests(requests, num_requests, doDryRun, useDev): # Update request with generator parameters update_answer = mcm.updateA('requests', mod_req) if update_answer['results']: - print "{0} created".format(answer['prepid']) + print "\033[0;32m{0} created\033[0;m".format(answer['prepid']) else: - print "{0} created but generator parameters not set".format( + print "\033[0;33m{0} created but generator parameters not set\033[0;m".format( answer['prepid']) else: if reqFields.useDataSetname(): - print "{0} failed to be created".format( + print "\033[0;31m{0} failed to be created\033[0;m".format( reqFields.getDataSetName()) else: - print "A request has failed to be created" + print "\033[0;31mA request has failed to be created\033[0;m" else: if reqFields.useDataSetName(): - print "{0} not created".format(reqFields.getDataSetName()) + print "\033[0;31m{0} not created\033[0;m".format( + reqFields.getDataSetName()) else: - print "request not created" + print "\033[0;31mrequest not created\033[0;m" pprint.pprint(new_req) def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): @@ -399,10 +400,10 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): # Get request from McM if isLHErequest: if not reqFields.useDataSetName(): - print "\033[1;31mDataset name missing\033[1;m" + print "\033[0;31mDataset name missing\033[0;m" continue elif not reqFields.useCamp(): - print "\033[1;31m{0} modification failed. Must provide campaign.\033[1;m".format( + print "\033[0;31m{0} modification failed. Must provide campaign.\033[0;m".format( reqFields.getDataSetName()) continue query_string = "dataset_name={0}&member_of_campaign={1}".format( @@ -415,21 +416,21 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): failed_to_get = False break if failed_to_get: - print "\033[1;31m{0} modification failed. Could not get request from McM.\033[1;m".format( + print "\033[0;31m{0} modification failed. Could not get request from McM.\033[0;m".format( reqFields.getDataSetName()) continue if len(mod_req_list) > 1: - print "\033[1;31m{0} modification failed. Too many requests match query.\033[1;m".format( + print "\033[0;31m{0} modification failed. Too many requests match query.\033[0;m".format( reqFields.getDataSetName()) continue elif len(mod_req_list) == 0: - print "\033[1;31m{0} modification failed. No requests match query.\033[1;m".format( + print "\033[0;31m{0} modification failed. No requests match query.\033[0;m".format( reqFields.getDataSetName()) continue mod_req = mod_req_list[0] else: if not reqFields.usePrepId(): - print "\033[1;31mPrepId is missing.\033[1;m" + print "\033[0;31mPrepId is missing.\033[0;m" continue time.sleep(0.1) mod_req = mcm.getA('requests', reqFields.getPrepId()) @@ -488,24 +489,26 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): answer = mcm.updateA('requests', mod_req) # Update request if answer['results']: if not isLHErequest: - print "\033[1;32m{0} modified\033[1;m".format( + print "\033[0;32m{0} modified\033[0;m".format( reqFields.getPrepId()) else: - print "\033[1;32m{0} in {1} modified\033[1;m".format( + print "\033[0;32m{0} in {1} modified\033[0;m".format( reqFields.getDataSetName(), reqFields.getCamp()) else: if not isLHErequest: - print "\033[1;31m{0} failed to be modified\033[1;m".format( + print "\033[0;31m{0} failed to be modified\033[0;m".format( reqFields.getPrepId()) else: - print "\033[1;31m{0} failed to be modified\033[1;m".format( + print "\033[0;31m{0} failed to be modified\033[0;m".format( reqFields.getDataSetName()) else: if not isLHErequest: - print "{0} not modified".format(reqFields.getPrepId()) + print "\033[0;31m{0} not modified\033[0;m".format( + reqFields.getPrepId()) pprint.pprint(mod_req) else: - print "{0} not modified".format(reqFields.getDataSetName()) + print "\033[0;31m{0} not modified\033[0;m".format( + reqFields.getDataSetName()) pprint.pprint(mod_req) @@ -564,19 +567,20 @@ def cloneRequests(requests, num_requests, doDryRun, useDev, cloneId_): if not doDryRun: answer = mcm.clone(cloneId_, clone_req) # Clone request if answer['results']: - print "{0} created using clone".format(answer['prepid']) + print "\033[0;32m{0} created using clone\033[0;m".format( + answer['prepid']) else: if reqFields.useDataSetName(): - print "{0} failed to be created using clone".format( + print "\033[0;31m{0} failed to be created using clone\033[0;m".format( reqFields.getDataSetName()) else: - print "request failed to be created using clone" + print "\033[0;31mrequest failed to be created using clone\033[0;m" else: if reqFields.useDataSetName(): - print "{0} not created using clone".format( + print "\033[0;31m{0} not created using clone\033[0;m".format( reqFields.getDataSetName()) else: - print "request not created using clone" + print "\033[0;31mrequest not created using clone\033[0;m" pprint.pprint(clone_req) def main(): From d89d5cfff6f78008cf8dd28a1b76c481f391dd58 Mon Sep 17 00:00:00 2001 From: perrozzi Date: Sun, 29 Nov 2015 16:16:25 +0100 Subject: [PATCH 04/64] expand getRequests.py + example_commands.txt + copy_tarballs.py Conflicts: getRequests.py --- copy_tarballs.py | 64 ++++++++++++++++++++++ example_commands.txt | 29 ++++++++++ getRequests.py | 127 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 214 insertions(+), 6 deletions(-) create mode 100755 copy_tarballs.py create mode 100644 example_commands.txt diff --git a/copy_tarballs.py b/copy_tarballs.py new file mode 100755 index 0000000..ba52b14 --- /dev/null +++ b/copy_tarballs.py @@ -0,0 +1,64 @@ +import os,sys +from subprocess import Popen, PIPE + +exit_anyway_after_check = False +# exit_anyway_after_check = True + +inputs_dir ='/afs/cern.ch/work/p/perrozzi/private/git/Hbb/McMScripts/gridpacks/' + +# target_main = '/afs/cern.ch/user/p/perrozzi/eos/cms/store/group/phys_generator/cvmfs/gridpacks/slc6_amd64_gcc481/13TeV/powheg/V2/' +target_main = '/afs/cern.ch/user/p/perrozzi/eos/cms/store/group/phys_generator/cvmfs/gridpacks/slc6_amd64_gcc481/13TeV/madgraph/V5_2.3.2.2/' + +print 'target main folder',target_main +if not os.path.isdir('/afs/cern.ch/user/p/perrozzi/eos/cms/store/group/phys_generator/cvmfs/gridpacks/'): + print 'mount eos first!' + sys.exit(1) +else: + print 'eos mounted' + +version = "v1" +print 'version',version + +print 'input dir',inputs_dir +inputs = filter(None,os.popen('ls '+inputs_dir).read().split('\n')) + +existing_list = [] +existing_list2 = [] +trow_exception = False + +for input in inputs: + foldername = input.replace('_tarball','').replace('.tar.gz','').replace('.tar.xz','') + fullpath = target_main+"/"+foldername + fullpath_version = fullpath+"/"+version + print "checking version folder",version,"for",foldername,", check if it is empty" + if os.path.isdir(fullpath_version) and (len(os.listdir(fullpath_version))!=0): + print "file already inside",fullpath_version,"please change version" + existing_list.append(((fullpath_version+"/"+os.listdir(fullpath_version)[0]).replace('/afs/cern.ch/user/p/perrozzi/eos/cms','')).replace('//','/')) + trow_exception = True + +if(trow_exception): + print 'same files already existed, please check' + print existing_list + sys.exit(1) + +if exit_anyway_after_check: sys.exit(1) + +for input in inputs: + foldername = input.replace('_tarball','').replace('.tar.gz','').replace('.tar.xz','') + fullpath = target_main+"/"+foldername + fullpath_version = fullpath+"/"+version+"/" + print 'foldername',foldername.replace('/afs/cern.ch/user/p/perrozzi/eos/cms','') + print 'os.path.isdir('+fullpath.replace('/afs/cern.ch/user/p/perrozzi/eos/cms','')+')',os.path.isdir(fullpath) + if not os.path.isdir(fullpath): + os.makedirs(fullpath) + print 'os.path.isdir('+fullpath_version.replace('/afs/cern.ch/user/p/perrozzi/eos/cms','')+')',os.path.isdir(fullpath_version) + if not os.path.isdir(fullpath_version): + os.makedirs(fullpath_version) + + print("cp "+inputs_dir+"/"+input+" "+fullpath_version.replace('/afs/cern.ch/user/p/perrozzi/eos/cms','')+'/') + os.system("cp "+inputs_dir+"/"+input+" "+fullpath_version+'/') + existing_list2.append(((fullpath_version+'/'+os.listdir(fullpath_version)[0]).replace('/afs/cern.ch/user/p/perrozzi/eos/cms/store/group/phys_generator/cvmfs','/cvmfs/cms.cern.ch/phys_generator')).replace('//','/')) + +print 'list of copied files' +print existing_list2 + diff --git a/example_commands.txt b/example_commands.txt new file mode 100644 index 0000000..12ef51a --- /dev/null +++ b/example_commands.txt @@ -0,0 +1,29 @@ +# setup environment +cd /afs/cern.ch/work/p/perrozzi/private/git/Hbb/validations/CMSSW_7_1_20_patch2/src; cmsenv; cd - +cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess +cern-get-sso-cookie -u https://cms-pdmv-dev.cern.ch/mcm/ -o ~/private/dev-cookie.txt --krb --reprocess +voms-proxy-init -voms cms + +# list of requests (only prep-Ids) +python getRequests.py "actor=perrozzi&member_of_campaign=*LHE*&status=new" + +# list of requests (with more info, from dataset names to full info) +python getRequests.py -listattr 0 "actor=perrozzi&member_of_campaign=*GS*&status=new" +... +python getRequests.py -listattr 5 "actor=perrozzi&member_of_campaign=*GS*&status=new" + +# check list of requests +python checkRequests.py PrepIDList + +# new requests +python manageRequests.py -c name_of_campaign input.csv + +# clone existing requests +python manageRequests.py --clone PrepId_of_request_to_clone input.csv + +# modify existing requests +python manageRequests.py -m input.csv + +# validate chains +python validateChains.py PrepIDList + diff --git a/getRequests.py b/getRequests.py index aa20911..c9df1ac 100755 --- a/getRequests.py +++ b/getRequests.py @@ -20,6 +20,32 @@ from rest import * # Load class to access McM from requestClass import * # Load class to store request information +class bcolors: + MAGENTA = '\033[35m' + BLUE = '\033[34m' + GREEN = '\033[32m' + # RED = '\033[31m' + YELLOW = '\033[33m' + WHITE = '\033[1;37m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + Gray_like_Ghost = '\033[1;30m' + RED = '\033[1;31m' + Green_like_Grass = '\033[1;32m' + Yellow_like_Yolk = '\033[1;33m' + Blue_like_Blood = '\033[1;34m' + Magenta_like_Mimosa = '\033[1;35m' + CYAN = '\033[1;36m' + Crimson_like_Chianti = '\033[1;38m' + Highlighted_Red_like_Radish = '\033[1;41m' + Highlighted_Green_like_Grass = '\033[1;42m' + Highlighted_Brown_like_Bear = '\033[1;43m' + Highlighted_Blue_like_Blood = '\033[1;44m' + Highlighted_Magenta_like_Mimosa = '\033[1;45m' + Highlighted_Cyan_like_Caribbean = '\033[1;46m' + Highlighted_Gray_like_Ghost = '\033[1;47m' + Highlighted_Crimson_like_Chianti = '\033[1;48m' def getArguments(): parser = argparse.ArgumentParser( @@ -33,6 +59,8 @@ def getArguments(): help='Only get requests with positive time and size per event.') parser.add_argument('-c', action='store_true', dest='getChain', help='Return PrepID of chain.') + parser.add_argument('-listattr', dest='listAttr', type=int, default=-1, + help='List attributes for each PrepID. 0 (default) to 5 in increasing level of verbosity') parser.add_argument('-f', dest='format', type=int, default=0, help='Format of output. 0 (default) = input for scripts, 1 = human-readable, 2 = HTML') @@ -47,11 +75,94 @@ def checkFile(file_): sys.exit(1) -def getPrepIDList(query_string, getNew, getForValidation, getChain): +def getMcMlist(query_string,printout): useDev = False mcm = restful(dev=useDev) # Get McM connection - print query_string + if printout: + print 'MCM query string: {0}{1}{2}'.format(bcolors.MAGENTA, + query_string, bcolors.ENDC) req_list = mcm.getA('requests', query=query_string) + return req_list + +def getPrepIDListWithAttributes(query_string,listAttr): + req_list = getMcMlist(query_string,True) + print '\n' + print '======================================================================================================================================================================\n' + for req in req_list: + # print '======================================================================================================================================================================\n',\ + # '======================================================================================================================================================================' + print bcolors.MAGENTA +\ + 'prepid='+ bcolors.ENDC,req['prepid'],\ + ', '+bcolors.MAGENTA+'Dataset name='+ bcolors.ENDC,req['dataset_name'],\ + ', '+bcolors.MAGENTA+'Extension='+ bcolors.ENDC,req['extension'],\ + ', '+bcolors.MAGENTA+'Completed/Total events='+ bcolors.ENDC,str(req['completed_events'])+'/'+str(req['total_events']),\ + ''+ bcolors.ENDC + if listAttr > 0: + print bcolors.RED +\ + 'Approval='+ bcolors.ENDC,req['approval'],\ + ', '+bcolors.RED+'Status='+ bcolors.ENDC,req['status'],\ + ', '+bcolors.RED+'Time Event='+ bcolors.ENDC,req['time_event'],\ + ', '+bcolors.RED+'CMSSW Release='+ bcolors.ENDC,req['cmssw_release'],\ + ', '+bcolors.RED+'Priority='+ bcolors.ENDC,req['priority'],\ + ''+ bcolors.ENDC + if listAttr > 1: + if(len(req['generator_parameters'])>0): + print bcolors.GREEN +\ + 'Cross Section='+ bcolors.ENDC,req['generator_parameters'][0]['cross_section'],'pb',\ + ', '+bcolors.GREEN+'Filter efficiency='+ bcolors.ENDC,str(req['generator_parameters'][0]['filter_efficiency'])+' +/- '+str(req['generator_parameters'][0]['filter_efficiency_error']),\ + ', '+bcolors.GREEN+'Match efficiency='+ bcolors.ENDC,str(req['generator_parameters'][0]['match_efficiency'])+' +/- '+str(req['generator_parameters'][0]['match_efficiency_error']),\ + ''+ bcolors.ENDC + else: + print bcolors.GREEN +\ + 'Cross Section= -1 pb',\ + ', Filter efficiency= -1',\ + ', Match efficiency= -1',\ + ''+ bcolors.ENDC + print bcolors.CYAN +\ + 'Tags='+ bcolors.ENDC,str(req['tags']).replace("u'",'').replace("'",""),\ + ', '+bcolors.CYAN+'Generators='+ bcolors.ENDC,req['name_of_fragment'],\ + ', '+bcolors.CYAN+'Name of Fragment='+ bcolors.ENDC,req['name_of_fragment'],\ + ', '+bcolors.CYAN+'Notes='+ bcolors.ENDC,req['notes'],\ + ''+ bcolors.ENDC + if listAttr > 2: + print bcolors.BLUE +\ + 'Last Updater Name='+ bcolors.ENDC,req['history'][0]['updater']['author_name'],\ + '(',req['history'][0]['updater']['author_email'],')',\ + '\n'\ + + bcolors.Gray_like_Ghost +\ + 'McM View Link= https://cms-pdmv.cern.ch/mcm/requests?shown=2199023255551&prepid='+req['prepid'],\ + '\n'\ + 'McM Edit Link= https://cms-pdmv.cern.ch/mcm/edit?db_name=requests&prepid='+req['prepid'],\ + ''+ bcolors.ENDC + if listAttr > 3: + print bcolors.YELLOW +\ + 'Member of chain(s)' + for current_chain in req['member_of_chain']: + query_chains = "member_of_chain="+current_chain + # print "req['member_of_chain'][0]",query_chains + temp = sys.stdout + f = open('/dev/null', 'w') + sys.stdout = f + chained_prepIds=getMcMlist(query_chains,False) + sys.stdout = temp + prepid1 = [] + for req1 in chained_prepIds: + prepid1.append(req1['prepid']) + print current_chain+" : "+ bcolors.ENDC+str(prepid1).strip('[]').replace("u'",'').replace("'","") + print bcolors.Gray_like_Ghost +\ + 'McM View Link= https://cms-pdmv.cern.ch/mcm/chained_requests?shown=4095&prepid='+current_chain,\ + ''+ bcolors.YELLOW + if listAttr > 4: + print bcolors.WHITE +'Fragment code=\n'+\ + bcolors.Gray_like_Ghost +\ + req['fragment'],\ + ''+ bcolors.ENDC + print '\n' + print bcolors.ENDC + print '======================================================================================================================================================================\n\n',\ + +def getPrepIDList(query_string, getNew, getForValidation, getChain): + req_list = getMcMlist(query_string,True) event_sum = 0 out_list = [] @@ -123,10 +234,14 @@ def printList(list, format): def main(): args = getArguments() # Setup flags and get arguments - list = getPrepIDList(args.query, args.getNew, args.getForValidation, - args.getChain) - printList(list, args.format) - + print 'args.listAttr',args.listAttr + if args.listAttr < 0: + list = getPrepIDList(args.query, args.getNew, args.getForValidation, + args.getChain) + printList(list, args.format) + else: + dict = getPrepIDListWithAttributes(args.query,args.listAttr) + return From 2b14a0f2adba25651d3eaba70c55bb9a51525128 Mon Sep 17 00:00:00 2001 From: perrozzi Date: Sun, 29 Nov 2015 16:29:02 +0100 Subject: [PATCH 05/64] document expanded version of getRequests.py --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1bbb523..bb37537 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Scripts for McM =============== -Author: David G. Sheffield (Rutgers) +Author: David G. Sheffield (Rutgers), extension of getRequests.py by Luca Perrozzi (ETHZ) Scripts for creating and updating requests in McM. See https://twiki.cern.ch/twiki/bin/viewauth/CMS/PdmVMcMScript for more information on scripts. @@ -135,6 +135,10 @@ The script can sort for only requests with time per event and size per event set To obtain a list of chained requests from wmLHE requests you can use the `-c` flag. +NEW: a new funcionality has been added to dump much more information (with COLORS!), to be launched with the flag '--listattr'. The default is `-f 0` (Dataset name, Extension, Number of Completed/Total events). The level of verbosity can be increased to `-f 1` (Status, Time Event, CMSSW Release, Priority), `-f 2` (Cross Section, Filter efficiency, Matching efficiency, Tags, Generators, Name of Fragment, Notes), `-f 3` (Last Updater Name and Email, McM View and Edit Links), `-f 4` (Member of the chains including prepIds of the chained requests and direct McM chain link), and `-f 5` (Fragment code). Example: + +`python getRequests.py -listattr 5 "actor=perrozzi&member_of_campaign=*GS*&status=new"` + Usage of testRequests.py ------------------------ From af3928f258b14295b1526940c9bf05da70bf92d0 Mon Sep 17 00:00:00 2001 From: perrozzi Date: Sun, 29 Nov 2015 16:53:51 +0100 Subject: [PATCH 06/64] expand getRequests.py + level 6, dump of request python dictionary --- getRequests.py | 143 ++++++++++++++++++++++++++----------------------- 1 file changed, 75 insertions(+), 68 deletions(-) diff --git a/getRequests.py b/getRequests.py index c9df1ac..493bd42 100755 --- a/getRequests.py +++ b/getRequests.py @@ -89,76 +89,83 @@ def getPrepIDListWithAttributes(query_string,listAttr): print '\n' print '======================================================================================================================================================================\n' for req in req_list: - # print '======================================================================================================================================================================\n',\ - # '======================================================================================================================================================================' - print bcolors.MAGENTA +\ - 'prepid='+ bcolors.ENDC,req['prepid'],\ - ', '+bcolors.MAGENTA+'Dataset name='+ bcolors.ENDC,req['dataset_name'],\ - ', '+bcolors.MAGENTA+'Extension='+ bcolors.ENDC,req['extension'],\ - ', '+bcolors.MAGENTA+'Completed/Total events='+ bcolors.ENDC,str(req['completed_events'])+'/'+str(req['total_events']),\ - ''+ bcolors.ENDC - if listAttr > 0: - print bcolors.RED +\ - 'Approval='+ bcolors.ENDC,req['approval'],\ - ', '+bcolors.RED+'Status='+ bcolors.ENDC,req['status'],\ - ', '+bcolors.RED+'Time Event='+ bcolors.ENDC,req['time_event'],\ - ', '+bcolors.RED+'CMSSW Release='+ bcolors.ENDC,req['cmssw_release'],\ - ', '+bcolors.RED+'Priority='+ bcolors.ENDC,req['priority'],\ + if listAttr > 5: # full dump of the request object, useful for debugging purpose + print bcolors.MAGENTA +\ + 'prepid='+ bcolors.ENDC,req['prepid'],\ ''+ bcolors.ENDC - if listAttr > 1: - if(len(req['generator_parameters'])>0): - print bcolors.GREEN +\ - 'Cross Section='+ bcolors.ENDC,req['generator_parameters'][0]['cross_section'],'pb',\ - ', '+bcolors.GREEN+'Filter efficiency='+ bcolors.ENDC,str(req['generator_parameters'][0]['filter_efficiency'])+' +/- '+str(req['generator_parameters'][0]['filter_efficiency_error']),\ - ', '+bcolors.GREEN+'Match efficiency='+ bcolors.ENDC,str(req['generator_parameters'][0]['match_efficiency'])+' +/- '+str(req['generator_parameters'][0]['match_efficiency_error']),\ - ''+ bcolors.ENDC - else: - print bcolors.GREEN +\ - 'Cross Section= -1 pb',\ - ', Filter efficiency= -1',\ - ', Match efficiency= -1',\ - ''+ bcolors.ENDC - print bcolors.CYAN +\ - 'Tags='+ bcolors.ENDC,str(req['tags']).replace("u'",'').replace("'",""),\ - ', '+bcolors.CYAN+'Generators='+ bcolors.ENDC,req['name_of_fragment'],\ - ', '+bcolors.CYAN+'Name of Fragment='+ bcolors.ENDC,req['name_of_fragment'],\ - ', '+bcolors.CYAN+'Notes='+ bcolors.ENDC,req['notes'],\ - ''+ bcolors.ENDC - if listAttr > 2: - print bcolors.BLUE +\ - 'Last Updater Name='+ bcolors.ENDC,req['history'][0]['updater']['author_name'],\ - '(',req['history'][0]['updater']['author_email'],')',\ - '\n'\ - + bcolors.Gray_like_Ghost +\ - 'McM View Link= https://cms-pdmv.cern.ch/mcm/requests?shown=2199023255551&prepid='+req['prepid'],\ - '\n'\ - 'McM Edit Link= https://cms-pdmv.cern.ch/mcm/edit?db_name=requests&prepid='+req['prepid'],\ - ''+ bcolors.ENDC - if listAttr > 3: - print bcolors.YELLOW +\ - 'Member of chain(s)' - for current_chain in req['member_of_chain']: - query_chains = "member_of_chain="+current_chain - # print "req['member_of_chain'][0]",query_chains - temp = sys.stdout - f = open('/dev/null', 'w') - sys.stdout = f - chained_prepIds=getMcMlist(query_chains,False) - sys.stdout = temp - prepid1 = [] - for req1 in chained_prepIds: - prepid1.append(req1['prepid']) - print current_chain+" : "+ bcolors.ENDC+str(prepid1).strip('[]').replace("u'",'').replace("'","") - print bcolors.Gray_like_Ghost +\ - 'McM View Link= https://cms-pdmv.cern.ch/mcm/chained_requests?shown=4095&prepid='+current_chain,\ - ''+ bcolors.YELLOW - if listAttr > 4: - print bcolors.WHITE +'Fragment code=\n'+\ - bcolors.Gray_like_Ghost +\ - req['fragment'],\ + print str(req).replace("u'",'') + print '' + else: + # print '======================================================================================================================================================================\n',\ + # '======================================================================================================================================================================' + print bcolors.MAGENTA +\ + 'prepid='+ bcolors.ENDC,req['prepid'],\ + ', '+bcolors.MAGENTA+'Dataset name='+ bcolors.ENDC,req['dataset_name'],\ + ', '+bcolors.MAGENTA+'Extension='+ bcolors.ENDC,req['extension'],\ + ', '+bcolors.MAGENTA+'Completed/Total events='+ bcolors.ENDC,str(req['completed_events'])+'/'+str(req['total_events']),\ ''+ bcolors.ENDC - print '\n' - print bcolors.ENDC + if listAttr > 0: + print bcolors.RED +\ + 'Approval='+ bcolors.ENDC,req['approval'],\ + ', '+bcolors.RED+'Status='+ bcolors.ENDC,req['status'],\ + ', '+bcolors.RED+'Time Event='+ bcolors.ENDC,req['time_event'],\ + ', '+bcolors.RED+'CMSSW Release='+ bcolors.ENDC,req['cmssw_release'],\ + ', '+bcolors.RED+'Priority='+ bcolors.ENDC,req['priority'],\ + ''+ bcolors.ENDC + if listAttr > 1: + if(len(req['generator_parameters'])>0): + print bcolors.GREEN +\ + 'Cross Section='+ bcolors.ENDC,req['generator_parameters'][0]['cross_section'],'pb',\ + ', '+bcolors.GREEN+'Filter efficiency='+ bcolors.ENDC,str(req['generator_parameters'][0]['filter_efficiency'])+' +/- '+str(req['generator_parameters'][0]['filter_efficiency_error']),\ + ', '+bcolors.GREEN+'Match efficiency='+ bcolors.ENDC,str(req['generator_parameters'][0]['match_efficiency'])+' +/- '+str(req['generator_parameters'][0]['match_efficiency_error']),\ + ''+ bcolors.ENDC + else: + print bcolors.GREEN +\ + 'Cross Section= -1 pb',\ + ', Filter efficiency= -1',\ + ', Match efficiency= -1',\ + ''+ bcolors.ENDC + print bcolors.CYAN +\ + 'Tags='+ bcolors.ENDC,str(req['tags']).replace("u'",'').replace("'",""),\ + ', '+bcolors.CYAN+'Generators='+ bcolors.ENDC,req['name_of_fragment'],\ + ', '+bcolors.CYAN+'Name of Fragment='+ bcolors.ENDC,req['name_of_fragment'],\ + ', '+bcolors.CYAN+'Notes='+ bcolors.ENDC,req['notes'],\ + ''+ bcolors.ENDC + if listAttr > 2: + print bcolors.BLUE +\ + 'Last Updater Name='+ bcolors.ENDC,req['history'][0]['updater']['author_name'],\ + '(',req['history'][0]['updater']['author_email'],')',\ + '\n'\ + + bcolors.Gray_like_Ghost +\ + 'McM View Link= https://cms-pdmv.cern.ch/mcm/requests?shown=2199023255551&prepid='+req['prepid'],\ + '\n'\ + 'McM Edit Link= https://cms-pdmv.cern.ch/mcm/edit?db_name=requests&prepid='+req['prepid'],\ + ''+ bcolors.ENDC + if listAttr > 3: + print bcolors.YELLOW +\ + 'Member of chain(s)' + for current_chain in req['member_of_chain']: + query_chains = "member_of_chain="+current_chain + # print "req['member_of_chain'][0]",query_chains + temp = sys.stdout + f = open('/dev/null', 'w') + sys.stdout = f + chained_prepIds=getMcMlist(query_chains,False) + sys.stdout = temp + prepid1 = [] + for req1 in chained_prepIds: + prepid1.append(req1['prepid']) + print current_chain+" : "+ bcolors.ENDC+str(prepid1).strip('[]').replace("u'",'').replace("'","") + print bcolors.Gray_like_Ghost +\ + 'McM View Link= https://cms-pdmv.cern.ch/mcm/chained_requests?shown=4095&prepid='+current_chain,\ + ''+ bcolors.YELLOW + if listAttr > 4: + print bcolors.WHITE +'Fragment code=\n'+\ + bcolors.Gray_like_Ghost +\ + req['fragment'],\ + ''+ bcolors.ENDC + print bcolors.ENDC + print '======================================================================================================================================================================\n\n',\ def getPrepIDList(query_string, getNew, getForValidation, getChain): From f169457c4f133aa19c1b9d4d1a88b9e4fcdb1953 Mon Sep 17 00:00:00 2001 From: perrozzi Date: Sun, 29 Nov 2015 16:55:15 +0100 Subject: [PATCH 07/64] expand getRequests.py + level 6, dump of request python dictionary --- getRequests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getRequests.py b/getRequests.py index 493bd42..de1408a 100755 --- a/getRequests.py +++ b/getRequests.py @@ -93,7 +93,7 @@ def getPrepIDListWithAttributes(query_string,listAttr): print bcolors.MAGENTA +\ 'prepid='+ bcolors.ENDC,req['prepid'],\ ''+ bcolors.ENDC - print str(req).replace("u'",'') + print str(req).replace("u'","'") print '' else: # print '======================================================================================================================================================================\n',\ From bce4b811dad6adbd7ff873a50e2fac164c29b8ba Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 2 Dec 2015 19:54:51 +0100 Subject: [PATCH 08/64] Removed copy_tarballs.py and example_commands.txt --- copy_tarballs.py | 64 -------------------------------------------- example_commands.txt | 29 -------------------- 2 files changed, 93 deletions(-) delete mode 100755 copy_tarballs.py delete mode 100644 example_commands.txt diff --git a/copy_tarballs.py b/copy_tarballs.py deleted file mode 100755 index ba52b14..0000000 --- a/copy_tarballs.py +++ /dev/null @@ -1,64 +0,0 @@ -import os,sys -from subprocess import Popen, PIPE - -exit_anyway_after_check = False -# exit_anyway_after_check = True - -inputs_dir ='/afs/cern.ch/work/p/perrozzi/private/git/Hbb/McMScripts/gridpacks/' - -# target_main = '/afs/cern.ch/user/p/perrozzi/eos/cms/store/group/phys_generator/cvmfs/gridpacks/slc6_amd64_gcc481/13TeV/powheg/V2/' -target_main = '/afs/cern.ch/user/p/perrozzi/eos/cms/store/group/phys_generator/cvmfs/gridpacks/slc6_amd64_gcc481/13TeV/madgraph/V5_2.3.2.2/' - -print 'target main folder',target_main -if not os.path.isdir('/afs/cern.ch/user/p/perrozzi/eos/cms/store/group/phys_generator/cvmfs/gridpacks/'): - print 'mount eos first!' - sys.exit(1) -else: - print 'eos mounted' - -version = "v1" -print 'version',version - -print 'input dir',inputs_dir -inputs = filter(None,os.popen('ls '+inputs_dir).read().split('\n')) - -existing_list = [] -existing_list2 = [] -trow_exception = False - -for input in inputs: - foldername = input.replace('_tarball','').replace('.tar.gz','').replace('.tar.xz','') - fullpath = target_main+"/"+foldername - fullpath_version = fullpath+"/"+version - print "checking version folder",version,"for",foldername,", check if it is empty" - if os.path.isdir(fullpath_version) and (len(os.listdir(fullpath_version))!=0): - print "file already inside",fullpath_version,"please change version" - existing_list.append(((fullpath_version+"/"+os.listdir(fullpath_version)[0]).replace('/afs/cern.ch/user/p/perrozzi/eos/cms','')).replace('//','/')) - trow_exception = True - -if(trow_exception): - print 'same files already existed, please check' - print existing_list - sys.exit(1) - -if exit_anyway_after_check: sys.exit(1) - -for input in inputs: - foldername = input.replace('_tarball','').replace('.tar.gz','').replace('.tar.xz','') - fullpath = target_main+"/"+foldername - fullpath_version = fullpath+"/"+version+"/" - print 'foldername',foldername.replace('/afs/cern.ch/user/p/perrozzi/eos/cms','') - print 'os.path.isdir('+fullpath.replace('/afs/cern.ch/user/p/perrozzi/eos/cms','')+')',os.path.isdir(fullpath) - if not os.path.isdir(fullpath): - os.makedirs(fullpath) - print 'os.path.isdir('+fullpath_version.replace('/afs/cern.ch/user/p/perrozzi/eos/cms','')+')',os.path.isdir(fullpath_version) - if not os.path.isdir(fullpath_version): - os.makedirs(fullpath_version) - - print("cp "+inputs_dir+"/"+input+" "+fullpath_version.replace('/afs/cern.ch/user/p/perrozzi/eos/cms','')+'/') - os.system("cp "+inputs_dir+"/"+input+" "+fullpath_version+'/') - existing_list2.append(((fullpath_version+'/'+os.listdir(fullpath_version)[0]).replace('/afs/cern.ch/user/p/perrozzi/eos/cms/store/group/phys_generator/cvmfs','/cvmfs/cms.cern.ch/phys_generator')).replace('//','/')) - -print 'list of copied files' -print existing_list2 - diff --git a/example_commands.txt b/example_commands.txt deleted file mode 100644 index 12ef51a..0000000 --- a/example_commands.txt +++ /dev/null @@ -1,29 +0,0 @@ -# setup environment -cd /afs/cern.ch/work/p/perrozzi/private/git/Hbb/validations/CMSSW_7_1_20_patch2/src; cmsenv; cd - -cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess -cern-get-sso-cookie -u https://cms-pdmv-dev.cern.ch/mcm/ -o ~/private/dev-cookie.txt --krb --reprocess -voms-proxy-init -voms cms - -# list of requests (only prep-Ids) -python getRequests.py "actor=perrozzi&member_of_campaign=*LHE*&status=new" - -# list of requests (with more info, from dataset names to full info) -python getRequests.py -listattr 0 "actor=perrozzi&member_of_campaign=*GS*&status=new" -... -python getRequests.py -listattr 5 "actor=perrozzi&member_of_campaign=*GS*&status=new" - -# check list of requests -python checkRequests.py PrepIDList - -# new requests -python manageRequests.py -c name_of_campaign input.csv - -# clone existing requests -python manageRequests.py --clone PrepId_of_request_to_clone input.csv - -# modify existing requests -python manageRequests.py -m input.csv - -# validate chains -python validateChains.py PrepIDList - From 114a994c41753ec2039631033c21bb8f3870e6f8 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 2 Dec 2015 20:13:33 +0100 Subject: [PATCH 09/64] Reformat Make horizontal rules shorter Add author Clean up formatting --- getRequests.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/getRequests.py b/getRequests.py index de1408a..2872e70 100755 --- a/getRequests.py +++ b/getRequests.py @@ -6,7 +6,8 @@ # # Script to get a list of requests from McM # -# author: David G. Sheffield (Rutgers) +# authors: David G. Sheffield (Rutgers) +# Luca Perrozzi (ETHZ) # ################################ @@ -31,8 +32,8 @@ class bcolors: BOLD = '\033[1m' UNDERLINE = '\033[4m' Gray_like_Ghost = '\033[1;30m' - RED = '\033[1;31m' - Green_like_Grass = '\033[1;32m' + RED = '\033[1;31m' + Green_like_Grass = '\033[1;32m' Yellow_like_Yolk = '\033[1;33m' Blue_like_Blood = '\033[1;34m' Magenta_like_Mimosa = '\033[1;35m' @@ -87,7 +88,7 @@ def getMcMlist(query_string,printout): def getPrepIDListWithAttributes(query_string,listAttr): req_list = getMcMlist(query_string,True) print '\n' - print '======================================================================================================================================================================\n' + print '================================================================================\n' for req in req_list: if listAttr > 5: # full dump of the request object, useful for debugging purpose print bcolors.MAGENTA +\ @@ -96,8 +97,6 @@ def getPrepIDListWithAttributes(query_string,listAttr): print str(req).replace("u'","'") print '' else: - # print '======================================================================================================================================================================\n',\ - # '======================================================================================================================================================================' print bcolors.MAGENTA +\ 'prepid='+ bcolors.ENDC,req['prepid'],\ ', '+bcolors.MAGENTA+'Dataset name='+ bcolors.ENDC,req['dataset_name'],\ @@ -165,9 +164,9 @@ def getPrepIDListWithAttributes(query_string,listAttr): req['fragment'],\ ''+ bcolors.ENDC print bcolors.ENDC - - print '======================================================================================================================================================================\n\n',\ - + + print '================================================================================\n\n',\ + def getPrepIDList(query_string, getNew, getForValidation, getChain): req_list = getMcMlist(query_string,True) @@ -241,14 +240,13 @@ def printList(list, format): def main(): args = getArguments() # Setup flags and get arguments - print 'args.listAttr',args.listAttr if args.listAttr < 0: list = getPrepIDList(args.query, args.getNew, args.getForValidation, args.getChain) printList(list, args.format) else: dict = getPrepIDListWithAttributes(args.query,args.listAttr) - + return From a23a61e3de5e1684f9bc27a78afb5ead0ea97f68 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 2 Dec 2015 20:26:34 +0100 Subject: [PATCH 10/64] Dump dictionary formatted with pprint --- getRequests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getRequests.py b/getRequests.py index 2872e70..3ed4e4d 100755 --- a/getRequests.py +++ b/getRequests.py @@ -94,7 +94,7 @@ def getPrepIDListWithAttributes(query_string,listAttr): print bcolors.MAGENTA +\ 'prepid='+ bcolors.ENDC,req['prepid'],\ ''+ bcolors.ENDC - print str(req).replace("u'","'") + pprint.pprint(req) print '' else: print bcolors.MAGENTA +\ From dae19a2e5dc9befaebf99faf992175614e3e8e77 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 2 Dec 2015 20:37:22 +0100 Subject: [PATCH 11/64] Remove grey and white colors that might not be visible in terminals --- getRequests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/getRequests.py b/getRequests.py index 3ed4e4d..a3bd2b8 100755 --- a/getRequests.py +++ b/getRequests.py @@ -135,11 +135,11 @@ def getPrepIDListWithAttributes(query_string,listAttr): 'Last Updater Name='+ bcolors.ENDC,req['history'][0]['updater']['author_name'],\ '(',req['history'][0]['updater']['author_email'],')',\ '\n'\ - + bcolors.Gray_like_Ghost +\ + + bcolors.ENDC +\ 'McM View Link= https://cms-pdmv.cern.ch/mcm/requests?shown=2199023255551&prepid='+req['prepid'],\ '\n'\ 'McM Edit Link= https://cms-pdmv.cern.ch/mcm/edit?db_name=requests&prepid='+req['prepid'],\ - ''+ bcolors.ENDC + '' if listAttr > 3: print bcolors.YELLOW +\ 'Member of chain(s)' @@ -155,12 +155,12 @@ def getPrepIDListWithAttributes(query_string,listAttr): for req1 in chained_prepIds: prepid1.append(req1['prepid']) print current_chain+" : "+ bcolors.ENDC+str(prepid1).strip('[]').replace("u'",'').replace("'","") - print bcolors.Gray_like_Ghost +\ + print bcolors.ENDC +\ 'McM View Link= https://cms-pdmv.cern.ch/mcm/chained_requests?shown=4095&prepid='+current_chain,\ ''+ bcolors.YELLOW if listAttr > 4: - print bcolors.WHITE +'Fragment code=\n'+\ - bcolors.Gray_like_Ghost +\ + print bcolors.Green_like_Grass +'Fragment code=\n'+\ + bcolors.ENDC +\ req['fragment'],\ ''+ bcolors.ENDC print bcolors.ENDC From 603ac2ba0ae595979fdb435ace513efd9c1c48f5 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 2 Dec 2015 21:06:09 +0100 Subject: [PATCH 12/64] Get PWG from configuration file --- README.md | 2 +- manageRequests.py | 7 +++---- mcmscripts_config.py | 7 +++++++ 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 mcmscripts_config.py diff --git a/README.md b/README.md index bb37537..ff75a01 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Usage of manageRequests.py ### Setup PWG -The default PWG in manageRequests.py is set to XXX. To modify your default PWG, change the variable `defaultPWG` on line 23. Alternatively, you can include the flag `-p your_PWG`. +The default PWG in manageRequests.py is set by `mcmscripts_config.py`. To set it to your group, change the configuration file. Alternatively, you can include the flag `-p your_PWG`. ### Creating new requests diff --git a/manageRequests.py b/manageRequests.py index ce1990e..762b601 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -16,13 +16,12 @@ import csv import pprint import time +import mcmscripts_config sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import * # Load class to access McM from requestClass import * # Load class to store request information def getArguments(): - defaultPWG = 'XXX' # Change this line to your PWG, then -p flag is not needed - parser = argparse.ArgumentParser( description='Create, modify, and clone McM requests.') @@ -31,8 +30,8 @@ def getArguments(): parser.add_argument('-c', '--campaign', action='store', dest='campaign', metavar='name', help='Set member_of_campaign.') parser.add_argument('-p', '--pwg', action='store', dest='pwg', - default=defaultPWG, - help='Set PWG. Defaults to %(default)s. Change the variable defaultPWG to your PWG.') + default=mcmscripts_config.pwg, + help='Set PWG. Defaults to %(default)s. Change default in config.py') parser.add_argument('-m', '--modify', action='store_true', dest='doModify', help='Modify existing requests. The CSV file must contain the PrepIds of the requests to be modified.') parser.add_argument('--clone', action='store', dest='cloneId', default='', diff --git a/mcmscripts_config.py b/mcmscripts_config.py new file mode 100644 index 0000000..edbde59 --- /dev/null +++ b/mcmscripts_config.py @@ -0,0 +1,7 @@ +# Physics working group for manageRequests.py +# Options: B2G BPH BTW EGM EWK EXO FSQ FWD HCA HIG HIN JME L1T MUO QCD SMP SUS +# TAU TOP TRK TSG +pwg = 'XXX' + +# McM username +username = 'inewton' From ec7ce5e9b95c7652dc6143945824308160ab3647 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 2 Dec 2015 21:18:24 +0100 Subject: [PATCH 13/64] Add installation instructions for config file --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index ff75a01..8197a68 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,17 @@ Scripts * [**getMcMTestScript.sh**](#usage-of-getmcmtestscriptsh) Get test script from McM with some modifications. * [**getTimeSize.sh**](#usage-of-gettimesizesh) Extract time/size per event from cmsRun job report. +Installation +------------ + +Once you've created a clone of this repository run the command + +``` +git update-index --assume-unchanged mcmscripts_config.py +``` + +to avoid committing any changes to your personal configuration file. Then modify the default values in mcmscripts_config.py to use your physics working group and username as default. + Get cookies ----------- From 0e9d64c162eae4c68fa8047bd1a3cd70fb7eeab8 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Fri, 4 Dec 2015 17:29:20 +0100 Subject: [PATCH 14/64] Add ability to manage tags in McM Add McMTag members to requestClass.py Use tags creating or modifying requests Update README.md --- README.md | 1 + manageRequests.py | 22 +++++++++++++++++++--- requestClass.py | 8 ++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8197a68..e8e5bad 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ Information for requests is provided in a CSV file. The script reads the first l * Gridpack location * Gridpack cards URL * Notes +* McM tag It will also recognize some alternative names. If there is a field title that the script does not recognize, it will complain. The script will ignore columns with the headers "JobId", "Local gridpack location", and "Local LHE" as they are used to supply information to other scripts but do not contain information for McM. diff --git a/manageRequests.py b/manageRequests.py index 762b601..7c9728b 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -42,6 +42,8 @@ def getArguments(): help='Use dev/test instance.') parser.add_argument('-l', '--lhe', action='store_true', dest='isLHErequest', help='Check dataset when modifying requests. Fail and do not modify name if they conflict. Use for updating GS requests chained to wmLHE and pLHE requests.') + parser.add_argument('-t', '--tags', action='append', dest='McMTags', + metavar='tags', help='Tags to append to request in McM.') args_ = parser.parse_args() return args_ @@ -86,7 +88,7 @@ def exitDuplicateField(file_in_, field_): def getFields(csvfile_, file_in_): # List of indices for each field in CSV file list = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1] + -1, -1, -1, -1, -1] header = csv.reader(csvfile_).next() for ind, field in enumerate(header): if field in ['Dataset name', 'Dataset Name', 'Dataset', 'dataset']: @@ -192,6 +194,10 @@ def getFields(csvfile_, file_in_): if list[20] > -1: exitDuplicateField(file_in_, "Notes") list[20] = ind + elif field in ['McM tag', 'McM Tag', 'McM tags', 'McM Tags']: + if list[21] > -1: + exitDuplicateField(file_in_, "McM tags") + list[21] = ind elif field in ['JobId', 'Local gridpack location', 'Local LHE', 'LHE']: continue else: @@ -234,7 +240,7 @@ def createLHEProducer(gridpack, cards): """.format(cards) return code -def fillFields(csvfile, fields, campaign, PWG, notCreate_): +def fillFields(csvfile, fields, campaign, PWG, notCreate_, McMTags): requests = [] # List containing request objects num_requests = 0 for row in csv.reader(csvfile): @@ -301,6 +307,10 @@ def fillFields(csvfile, fields, campaign, PWG, notCreate_): tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], "")) if fields[20] > -1: tmpReq.setNotes(row[fields[20]]) + if fields[21] > -1: + tmpReq.setMcMTag(row[fields[21]].split(" ")) + elif McMTags is not None: + tmpReq.setMcMTag(McMTags) requests.append(tmpReq) return requests, num_requests @@ -346,6 +356,8 @@ def createRequests(requests, num_requests, doDryRun, useDev): new_req['process_string'] = reqFields.getProcessString() if reqFields.useNotes(): new_req['notes'] = reqFields.getNotes() + if reqFields.useMcMTag(): + new_req['tags'] = reqFields.getMcMTag() if not doDryRun: answer = mcm.putA('requests', new_req) # Create request @@ -483,6 +495,8 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): mod_req['process_string'] = reqFields.getProcessString() if reqFields.useNotes(): mod_req['notes'] = reqFields.getNotes() + if reqFields.useMcMTag(): + mod_req['tags'] += reqFields.getMcMTag() if not doDryRun: answer = mcm.updateA('requests', mod_req) # Update request @@ -562,6 +576,8 @@ def cloneRequests(requests, num_requests, doDryRun, useDev, cloneId_): clone_req['process_string'] = reqFields.getProcessString() if reqFields.useNotes(): clone_req['notes'] = reqFields.getNotes() + if reqFields.useMcMTag(): + clone_req['tags'] += reqFields.getMcMTag() if not doDryRun: answer = mcm.clone(cloneId_, clone_req) # Clone request @@ -597,7 +613,7 @@ def main(): fields = getFields(csvfile, args.file_in) # Get list of field indices # Fill list of request objects with fields from CSV and get number of requests requests, num_requests = fillFields(csvfile, fields, args.campaign, - args.pwg, notCreate) + args.pwg, notCreate, args.McMTags) if args.doModify: # Modify existing requests diff --git a/requestClass.py b/requestClass.py index d40b8f3..6f61439 100644 --- a/requestClass.py +++ b/requestClass.py @@ -31,6 +31,7 @@ def __init__(self): self.useProcessString_ = False self.useJobID_ = False self.useNotes_ = False + self.useMcMTag_ = False def setDataSetName(self,x): self.DataSetName_ = x @@ -95,6 +96,9 @@ def setJobID(self,x): def setNotes(self, x): self.Notes_ = x self.useNotes_ = True + def setMcMTag(self, x): + self.McMTag_ = x + self.useMcMTag_ = True def getDataSetName(self): return self.DataSetName_ @@ -138,6 +142,8 @@ def getJobID(self): return self.JobID_ def getNotes(self): return self.Notes_ + def getMcMTag(self): + return self.McMTag_ def useDataSetName(self): return self.useDataSetName_ @@ -181,3 +187,5 @@ def useJobID(self): return self.useJobID_ def useNotes(self): return self.useNotes_ + def useMcMTag(self): + return self.useMcMTag_ From 89063df1541356c59bfb1599956deefe5dd338bf Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 9 Dec 2015 17:02:04 +0100 Subject: [PATCH 15/64] Add option to print out list of all PrepIDs Change arrow for HTML to us > instead of > --- getRequests.py | 65 ++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/getRequests.py b/getRequests.py index a3bd2b8..8e4092e 100755 --- a/getRequests.py +++ b/getRequests.py @@ -63,7 +63,7 @@ def getArguments(): parser.add_argument('-listattr', dest='listAttr', type=int, default=-1, help='List attributes for each PrepID. 0 (default) to 5 in increasing level of verbosity') parser.add_argument('-f', dest='format', type=int, default=0, - help='Format of output. 0 (default) = input for scripts, 1 = human-readable, 2 = HTML') + help='Format of output. 0 (default) = input for scripts, 1 = human-readable, 2 = HTML, 3 = write every PrepID') args_ = parser.parse_args() return args_ @@ -204,36 +204,43 @@ def isSequential(lastID, currentID): def printList(list, format): - arrow = "-" - comma = "," - if format == 1: - arrow = " ---> " - comma = ", " - elif format == 2: - arrow = " ---> " - comma = "
" - - lastID = "FIRST" - print_last = False - last_index = len(list) - 1 print "" - for i, PrepID in enumerate(list): - if isSequential(lastID, PrepID): - if i < last_index: - print_last = True - else: - sys.stdout.write("{0}{1}".format(arrow, PrepID)) - else: - if print_last: - sys.stdout.write("{0}{1}{2}{3}".format(arrow, lastID, comma, - PrepID)) - elif i > 0: - sys.stdout.write("{0}{1}".format(comma, PrepID)) + if format == 3: + for PrepID in list: + print PrepID + print "" + else: + arrow = "-" + comma = "," + if format == 1: + arrow = " ---> " + comma = ", " + elif format == 2: + arrow = " ---> " + comma = "
" + + lastID = "FIRST" + print_last = False + last_index = len(list) - 1 + for i, PrepID in enumerate(list): + if isSequential(lastID, PrepID): + if i < last_index: + print_last = True + else: + sys.stdout.write("{0}{1}".format(arrow, PrepID)) else: - sys.stdout.write("{0}".format(PrepID)) - print_last = False - lastID = PrepID - print "\n" + if print_last: + sys.stdout.write("{0}{1}{2}{3}".format(arrow, lastID, comma, + PrepID)) + elif i > 0: + sys.stdout.write("{0}{1}".format(comma, PrepID)) + else: + sys.stdout.write("{0}".format(PrepID)) + print_last = False + lastID = PrepID + print "" + + print "" return From f5baa9285f7f1ff1631f6c2c2402850bb7030368 Mon Sep 17 00:00:00 2001 From: Federico Ferri Date: Tue, 5 Jan 2016 18:35:11 +0100 Subject: [PATCH 16/64] switch off colors: automatically when the command is piped/redirected, or via an explicit option at will --- getRequests.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/getRequests.py b/getRequests.py index 8e4092e..c6abb1f 100755 --- a/getRequests.py +++ b/getRequests.py @@ -21,7 +21,7 @@ from rest import * # Load class to access McM from requestClass import * # Load class to store request information -class bcolors: +class bcolorClass: MAGENTA = '\033[35m' BLUE = '\033[34m' GREEN = '\033[32m' @@ -48,6 +48,18 @@ class bcolors: Highlighted_Gray_like_Ghost = '\033[1;47m' Highlighted_Crimson_like_Chianti = '\033[1;48m' + def _noColor(self): + for m in dir(self): + if not m.startswith('_'): + setattr(self, m, '') + + def __init__(self): + if not sys.stdout.isatty(): + self._noColor() + +bcolors = bcolorClass() + + def getArguments(): parser = argparse.ArgumentParser( description='Get a list of PrepIDs from McM based on a query.') @@ -62,6 +74,8 @@ def getArguments(): help='Return PrepID of chain.') parser.add_argument('-listattr', dest='listAttr', type=int, default=-1, help='List attributes for each PrepID. 0 (default) to 5 in increasing level of verbosity') + parser.add_argument('-bw', action='store_true', dest='noColors', default=False, + help='B/W output') parser.add_argument('-f', dest='format', type=int, default=0, help='Format of output. 0 (default) = input for scripts, 1 = human-readable, 2 = HTML, 3 = write every PrepID') @@ -252,6 +266,8 @@ def main(): args.getChain) printList(list, args.format) else: + if args.noColors: + bcolors._noColor() dict = getPrepIDListWithAttributes(args.query,args.listAttr) return From 6d948b7c982d57c30261f685600a9dbf5d491e79 Mon Sep 17 00:00:00 2001 From: Federico Ferri Date: Wed, 6 Jan 2016 00:06:50 +0100 Subject: [PATCH 17/64] read auto_bw from mcmscripts_config to determine whether to colorize only when stdout is connected to a terminal --- getRequests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/getRequests.py b/getRequests.py index c6abb1f..ed34a81 100755 --- a/getRequests.py +++ b/getRequests.py @@ -20,6 +20,7 @@ sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import * # Load class to access McM from requestClass import * # Load class to store request information +import mcmscripts_config class bcolorClass: MAGENTA = '\033[35m' @@ -54,7 +55,7 @@ def _noColor(self): setattr(self, m, '') def __init__(self): - if not sys.stdout.isatty(): + if mcmscripts_config.auto_bw and not sys.stdout.isatty(): self._noColor() bcolors = bcolorClass() From c286d9e2b2109bd1a989caf9afacb9538633b07e Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 6 Jan 2016 17:56:29 +0100 Subject: [PATCH 18/64] Add auto_bw variable to configuration file --- mcmscripts_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mcmscripts_config.py b/mcmscripts_config.py index edbde59..22824e4 100644 --- a/mcmscripts_config.py +++ b/mcmscripts_config.py @@ -5,3 +5,6 @@ # McM username username = 'inewton' + +# Automatically remove color when piping or redirecting output +auto_bw = False From 4672cac89e00e14155e9c8a770f7bc42dc955e3f Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 21 Jan 2016 17:54:15 +0100 Subject: [PATCH 19/64] Let manageRequests.py set beamspot and magField in sequences. No longer creates sequence when request is created. Creating it from scratch prevents the default sequence from being added. Instead, the default sequence is created and then modified along with the generator parameters. --- manageRequests.py | 33 ++++++++++++++++++++++++++++----- requestClass.py | 16 ++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/manageRequests.py b/manageRequests.py index 7c9728b..1039f0b 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -88,7 +88,7 @@ def exitDuplicateField(file_in_, field_): def getFields(csvfile_, file_in_): # List of indices for each field in CSV file list = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1] + -1, -1, -1, -1, -1, -1, -1] header = csv.reader(csvfile_).next() for ind, field in enumerate(header): if field in ['Dataset name', 'Dataset Name', 'Dataset', 'dataset']: @@ -198,6 +198,14 @@ def getFields(csvfile_, file_in_): if list[21] > -1: exitDuplicateField(file_in_, "McM tags") list[21] = ind + elif field in ['Sequences beamspot']: + if list[22] > -1: + exitDuplicateField(file_in_, "Sequences beamspot") + list[22] = ind + elif field in ['Sequences magField']: + if list[23] > -1: + exitDuplicateField(file_in_, "Sequences magField") + list[23] = ind elif field in ['JobId', 'Local gridpack location', 'Local LHE', 'LHE']: continue else: @@ -311,6 +319,10 @@ def fillFields(csvfile, fields, campaign, PWG, notCreate_, McMTags): tmpReq.setMcMTag(row[fields[21]].split(" ")) elif McMTags is not None: tmpReq.setMcMTag(McMTags) + if fields[22] > -1: + tmpReq.setSequencesBeamspot(row[fields[22]]) + if fields[23] > -1: + tmpReq.setSequencesMagField(row[fields[23]]) requests.append(tmpReq) return requests, num_requests @@ -349,9 +361,6 @@ def createRequests(requests, num_requests, doDryRun, useDev): new_req['size_event'] = reqFields.getSize() if reqFields.useGen(): new_req['generators'] = reqFields.getGen() - # Sequences might need to be added below with generator parameters - if reqFields.useSequencesCustomise(): - new_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() if reqFields.useProcessString(): new_req['process_string'] = reqFields.getProcessString() if reqFields.useNotes(): @@ -377,7 +386,13 @@ def createRequests(requests, num_requests, doDryRun, useDev): = reqFields.getMatchEff() mod_req['generator_parameters'][0]['match_efficiency_error'] \ = reqFields.getMatchEffErr() - # Update request with generator parameters + if reqFields.useSequencesCustomise(): + mod_req['sequences'][0]['customise'] = reqFields.getSequencesCustomise() + if reqFields.useSequencesBeamspot(): + mod_req['sequences'][0]['beamspot'] = reqFields.getSequencesBeamspot() + if reqFields.useSequencesMagField(): + mod_req['sequences'][0]['magField'] = reqFields.getSequencesMagField() + # Update request with generator parameters and sequences update_answer = mcm.updateA('requests', mod_req) if update_answer['results']: print "\033[0;32m{0} created\033[0;m".format(answer['prepid']) @@ -491,6 +506,10 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): if reqFields.useSequencesCustomise(): mod_req['sequences'][0]['customise'] \ = reqFields.getSequencesCustomise() + if reqFields.useSequencesBeamspot(): + mod_req['sequences'][0]['beamspot'] = reqFields.getSequencesBeamspot() + if reqFields.useSequencesMagField(): + mod_req['sequences'][0]['magField'] = reqFields.getSequencesMagField() if reqFields.useProcessString(): mod_req['process_string'] = reqFields.getProcessString() if reqFields.useNotes(): @@ -572,6 +591,10 @@ def cloneRequests(requests, num_requests, doDryRun, useDev, cloneId_): if reqFields.useSequencesCustomise(): clone_req['sequences'][0]['customise'] \ = reqFields.getSequencesCustomise() + if reqFields.useSequencesBeamspot(): + clone_req['sequences'][0]['beamspot'] = reqFields.getSequencesBeamspot() + if reqFields.useSequencesMagField(): + clone_req['sequences'][0]['magField'] = reqFields.getSequencesMagField() if reqFields.useProcessString(): clone_req['process_string'] = reqFields.getProcessString() if reqFields.useNotes(): diff --git a/requestClass.py b/requestClass.py index 6f61439..a430812 100644 --- a/requestClass.py +++ b/requestClass.py @@ -28,6 +28,8 @@ def __init__(self): self.useCamp_ = False self.usePrepId_ = False self.useSequencesCustomise_ = False + self.useSequencesBeamspot_ = False + self.useSequencesMagField_ = False self.useProcessString_ = False self.useJobID_ = False self.useNotes_ = False @@ -87,6 +89,12 @@ def setPrepId(self,x): def setSequencesCustomise(self,x): self.SequencesCustomise_ = x self.useSequencesCustomise_ = True + def setSequencesBeamspot(self,x): + self.SequencesBeamspot_ = x + self.useSequencesBeamspot_ = True + def setSequencesMagField(self,x): + self.SequencesMagField_ = x + self.useSequencesMagField_ = True def setProcessString(self,x): self.ProcessString_ = x self.useProcessString_ = True @@ -136,6 +144,10 @@ def getPrepId(self): return self.PrepId_ def getSequencesCustomise(self): return self.SequencesCustomise_ + def getSequencesBeamspot(self): + return self.SequencesBeamspot_ + def getSequencesMagField(self): + return self.SequencesMagField_ def getProcessString(self): return self.ProcessString_ def getJobID(self): @@ -181,6 +193,10 @@ def usePrepId(self): return self.usePrepId_ def useSequencesCustomise(self): return self.useSequencesCustomise_ + def useSequencesBeamspot(self): + return self.useSequencesBeamspot_ + def useSequencesMagField(self): + return self.useSequencesMagField_ def useProcessString(self): return self.useProcessString_ def useJobID(self): From 6b7e55bc920535540c78fed46e3146d37c0963ee Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 21 Jan 2016 18:10:57 +0100 Subject: [PATCH 20/64] Ignore case in CSV headers. Add new fields to testRequests.py to match manageRequests.py. --- manageRequests.py | 79 ++++++++++++++++---------------------- testRequests.py | 97 ++++++++++++++++++++++++++--------------------- 2 files changed, 87 insertions(+), 89 deletions(-) diff --git a/manageRequests.py b/manageRequests.py index 1039f0b..f40d130 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -91,122 +91,109 @@ def getFields(csvfile_, file_in_): -1, -1, -1, -1, -1, -1, -1] header = csv.reader(csvfile_).next() for ind, field in enumerate(header): - if field in ['Dataset name', 'Dataset Name', 'Dataset', 'dataset']: + field = field.lower() + if field in ['dataset name', 'dataset']: #ensure no duplicate fields if list[0] > -1: exitDuplicateField(file_in_, "Dataset name") list[0] = ind - elif field in ['EOS', 'eos', 'Eos', 'MCDBID', 'mcdbid']: + elif field in ['eos', 'mcdbid']: if list[1] > -1: exitDuplicateField(file_in_, "EOS") list[1] = ind - elif field in ['Cross section [pb]', 'Cross section', - 'Cross section (pb)', 'Cross Section', - 'Cross Section [pb]', 'Cross Section (pb)', 'CS', - 'CS [pb]', 'CS (pb)', 'Xsec', 'Xsec [pb]', 'Xsec (pb)']: + elif field in ['cross section [pb]', 'cross section (pb)', + 'cross section', 'cs', 'cs [pb]', 'cs (pb)', 'xsec', + 'xsec [pb]', 'xsec (pb)']: if list[2] > -1: - exitDuplicateField(file_in_, "Cross section") + exitDuplicateField(file_in_, "Cross section [pb]") list[2] = ind - elif field in ['Total events', 'Total Events', 'Events', 'events', - 'total events', 'Number of Events']: + elif field in ['total events', 'events', 'number of events']: if list[3] > -1: exitDuplicateField(file_in_, "Total events") list[3] = ind - elif field in ['Fragment name', 'Fragment Name', - 'Generator fragment name', 'Generator Fragment Name', - 'Fragment', 'fragment']: + elif field in ['fragment name', 'generator fragment name', 'fragment']: if list[4] > -1: exitDuplicateField(file_in_, "Fragment name") list[4] = ind - elif field in ['Time per event [s]', 'Time per event', - 'Time per event (s)', 'Time per Event', - 'Time per Event [s]', 'Time per Event (s)', 'Time', - 'Time [s]', 'Time (s)', 'time', 'time [s]', 'time (s)']: + elif field in ['time per event [s]', 'time per event', + 'time per event (s)', 'time', 'time [s]', 'time (s)']: if list[5] > -1: exitDuplicateField(file_in_, "Time per event [s]") list[5] = ind - elif field in ['Size per event [kB]', 'Size per event', - 'Size per event (kB)', 'Size per Event', - 'Size per Event [kB]', 'Size per Event (kB)', 'size', - 'size [kB]', 'size (kB)']: + elif field in ['size per event [kb]', 'size per event', + 'size per event (kb)', 'size', 'size [kb]', 'size (kb)']: if list[6] > -1: exitDuplicateField(file_in_, "Size per event [kB]") list[6] = ind - elif field in ['Tag', 'tag', 'Fragment Tag', 'Fragment tag', - 'fragment tag', 'sha', 'SHA', 'SHA-1', 'sha-1']: + elif field in ['tag', 'fragment tag', 'sha', 'sha-1']: if list[7] > -1: exitDuplicateField(file_in_, "Fragment tag") list[7] = ind - elif field in ['Generator', 'generator']: + elif field in ['generator', 'generators']: if list[8] > -1: exitDuplicateField(file_in_, "Generator") list[8] = ind - elif field in ['Filter efficiency', 'FilterEfficiency', - 'filter efficiency']: + elif field in ['filter efficiency']: if list[9] > -1: exitDuplicateField(file_in_, "Filter efficiency") list[9] = ind - elif field in ['Filter efficiency error', 'Filter Efficiency Error', - 'filter efficiency error']: + elif field in ['filter efficiency error']: if list[10] > -1: exitDuplicateField(file_in_, "Filter efficiency error") list[10] = ind - elif field in ['Match efficiency', 'Match Efficiency', - 'match efficiency']: + elif field in ['match efficiency']: if list[11] > -1: exitDuplicateField(file_in_, "Match efficiency") list[11] = ind - elif field in ['Match efficiency error', 'Match Efficiency Error', - 'match efficiency error']: + elif field in ['match efficiency error']: if list[12] > -1: exitDuplicateField(file_in_, "Match efficiency error") list[12] = ind - elif field in ['PWG', 'pwg']: + elif field in ['pwg']: if list[13] > -1: exitDuplicateField(file_in_, "PWG") list[13] = ind - elif field in ['Campaign', 'campaign', 'Member of Campaign', - 'Member of campaign', 'member of campaign']: + elif field in ['campaign', 'member of campaign']: if list[14] > -1: exitDuplicateField(file_in_, "Member of campaign") list[14] = ind - elif field in ['PrepId', 'PrepID', 'PREPID', 'prepid']: + elif field in ['prepid']: if list[15] > -1: - exitDuplicateField(file_in_, "PrepId") + exitDuplicateField(file_in_, "PrepID") list[15] = ind - elif field in ['Sequences customise', 'Sequences customize']: + elif field in ['sequences customise', 'sequences customize']: if list[16] > -1: exitDuplicateField(file_in_, "Sequences customise") list[16] = ind - elif field in ['Process string', 'Process String']: + elif field in ['process string']: if list[17] > -1: exitDuplicateField(file_in_, "Process string") list[17] = ind - elif field in ['Gridpack location', 'Gridpack']: + elif field in ['gridpack location', 'gridpack']: if list[18] > -1: exitDuplicateField(file_in_, "Gridpack location") list[18] = ind - elif field in ['Gridpack cards URL', 'Cards URL', - 'Gridpack cards location', 'Cards location']: + elif field in ['gridpack cards url', 'cards url', + 'gridpack cards location', 'cards location']: if list[19] > -1: exitDuplicateField(file_in_, "Gridpack cards URL") list[19] = ind - elif field in ['Notes', 'notes']: + elif field in ['notes']: if list[20] > -1: exitDuplicateField(file_in_, "Notes") list[20] = ind - elif field in ['McM tag', 'McM Tag', 'McM tags', 'McM Tags']: + elif field in ['mcm tag', 'mcm tags']: if list[21] > -1: exitDuplicateField(file_in_, "McM tags") list[21] = ind - elif field in ['Sequences beamspot']: + elif field in ['sequences beamspot']: if list[22] > -1: exitDuplicateField(file_in_, "Sequences beamspot") list[22] = ind - elif field in ['Sequences magField']: + elif field in ['sequences magField']: if list[23] > -1: exitDuplicateField(file_in_, "Sequences magField") list[23] = ind - elif field in ['JobId', 'Local gridpack location', 'Local LHE', 'LHE']: + elif field in ['jobid', 'local gridpack location', 'local lhe', 'lhe']: continue else: print "Error: The field {0} is not valid.".format(field) diff --git a/testRequests.py b/testRequests.py index 2827597..ce07079 100755 --- a/testRequests.py +++ b/testRequests.py @@ -160,111 +160,114 @@ def getFields(csvfile): -1, -1, -1, -1] header = csv.reader(csvfile).next() for ind, field in enumerate(header): - if field in ['Dataset name', 'Dataset Name', 'Dataset', 'dataset']: + field = field.lower() + if field in ['dataset name', 'dataset']: #ensure no duplicate fields if list[0] > -1: exitDuplicateField(file_in_, "Dataset name") list[0] = ind - elif field in ['EOS', 'eos', 'Eos', 'MCDBID', 'mcdbid']: + elif field in ['eos', 'mcdbid']: if list[1] > -1: exitDuplicateField(file_in_, "EOS") list[1] = ind - elif field in ['Cross section [pb]', 'Cross section', - 'Cross section (pb)', 'Cross Section', - 'Cross Section [pb]', 'Cross Section (pb)', 'CS', - 'CS [pb]', 'CS (pb)', 'Xsec', 'Xsec [pb]', 'Xsec (pb)']: + elif field in ['cross section [pb]', 'cross section', + 'cross section (pb)', 'cs', 'cs [pb]', 'cs (pb)', 'xsec', + 'xsec [pb]', 'xsec (pb)']: if list[2] > -1: - exitDuplicateField(file_in_, "Cross section") + exitDuplicateField(file_in_, "Cross section [pb]") list[2] = ind - elif field in ['Total events', 'Total Events', 'Events', 'events', - 'total events', 'Number of Events']: + elif field in ['total events', 'events', 'number of events']: if list[3] > -1: exitDuplicateField(file_in_, "Total events") list[3] = ind - elif field in ['Fragment name', 'Fragment Name', - 'Generator fragment name', 'Generator Fragment Name', - 'Fragment', 'fragment']: + elif field in ['fragment name', 'generator fragment name', 'fragment']: if list[4] > -1: exitDuplicateField(file_in_, "Fragment name") list[4] = ind - elif field in ['Time per event [s]', 'Time per event', - 'Time per event (s)', 'Time per Event', - 'Time per Event [s]', 'Time per Event (s)', 'Time', - 'Time [s]', 'Time (s)', 'time', 'time [s]', 'time (s)']: + elif field in ['time per event [s]', 'time per event', + 'time per event (s)', 'time', 'time [s]', 'time (s)']: if list[5] > -1: exitDuplicateField(file_in_, "Time per event [s]") list[5] = ind - elif field in ['Size per event [kB]', 'Size per event', - 'Size per event (kB)', 'Size per Event', - 'Size per Event [kB]', 'Size per Event (kB)', - 'size', 'size [kB]', 'size (kB)']: + elif field in ['size per event [kb]', 'size per event', + 'size per event (kb)', 'size', 'size [kb]', 'size (kb)']: if list[6] > -1: exitDuplicateField(file_in_, "Size per event [kB]") list[6] = ind - elif field in ['Tag', 'tag', 'Fragment Tag', 'Fragment tag', - 'fragment tag', 'sha', 'SHA', 'SHA-1', 'sha-1']: + elif field in ['tag', 'fragment tag', 'sha', 'sha-1']: if list[7] > -1: exitDuplicateField(file_in_, "Fragment tag") list[7] = ind - elif field in ['Generator', 'generator']: + elif field in ['generator', 'generators']: if list[8] > -1: exitDuplicateField(file_in_, "Generator") list[8] = ind - elif field in ['Filter efficiency', 'FilterEfficiency', - 'filter efficiency']: + elif field in ['filter efficiency']: if list[9] > -1: exitDuplicateField(file_in_, "Filter efficiency") list[9] = ind - elif field in ['Filter efficiency error', 'Filter Efficiency Error', - 'filter efficiency error']: + elif field in ['filter efficiency error']: if list[10] > -1: exitDuplicateField(file_in_, "Filter efficiency error") list[10] = ind - elif field in ['Match efficiency', 'Match Efficiency', - 'match efficiency']: + elif field in ['match efficiency']: if list[11] > -1: exitDuplicateField(file_in_, "Match efficiency") list[11] = ind - elif field in ['Match efficiency error', 'Match Efficiency Error', - 'match efficiency error']: + elif field in ['match efficiency error']: if list[12] > -1: exitDuplicateField(file_in_, "Match efficiency error") list[12] = ind - elif field in ['PWG', 'pwg']: + elif field in ['pwg']: if list[13] > -1: exitDuplicateField(file_in_, "PWG") list[13] = ind - elif field in ['Campaign', 'campaign', 'Member of Campaign', - 'Member of campaign', 'member of campaign']: + elif field in ['campaign', 'member of campaign']: if list[14] > -1: exitDuplicateField(file_in_, "Member of campaign") list[14] = ind - elif field in ['PrepId', 'PrepID', 'PREPID', 'prepid']: + elif field in ['prepid']: if list[15] > -1: - exitDuplicateField(file_in_, "PrepId") + exitDuplicateField(file_in_, "PrepID") list[15] = ind - elif field in ['Sequences customise', 'Sequences customize']: + elif field in ['sequences customise', 'sequences customize']: if list[16] > -1: exitDuplicateField(file_in_, "Sequences customise") list[16] = ind - elif field in ['Process string', 'Process String']: + elif field in ['process string']: if list[17] > -1: exitDuplicateField(file_in_, "Process string") list[17] = ind - elif field in ['Gridpack location', 'Gridpack']: + elif field in ['gridpack location', 'gridpack']: if list[18] > -1: exitDuplicateField(file_in_, "Gridpack location") list[18] = ind - elif field in ['Gridpack cards URL', 'Cards URL', - 'Gridpack cards location', 'Cards location']: + elif field in ['gridpack cards url', 'cards url', + 'gridpack cards location', 'cards location']: if list[19] > -1: exitDuplicateField(file_in_, "Gridpack cards URL") list[19] = ind - elif field in ['JobId']: + elif field in ['jobid']: if list[20] > -1: exitDuplicateField(file_in_, "JobId") list[20] = ind - elif field in ['Local gridpack location', 'Local LHE', 'LHE']: + elif field in ['notes']: + if list[21] > -1: + exitDuplicateField(file_in_, "Notes") + list[21] = ind + elif field in ['mcm tag', 'mcm tags']: + if list[22] > -1: + exitDuplicateField(file_in_, "McM tags") + list[22] = ind + elif field in ['sequences beamspot']: + if list[23] > -1: + exitDuplicateField(file_in_, "Sequences beamspot") + list[23] = ind + elif field in ['sequences magField']: + if list[24] > -1: + exitDuplicateField(file_in_, "Sequences magField") + list[24] = ind + elif field in ['local gridpack location', 'Local lhe', 'lhe']: continue else: print "Error: The field {0} is not valid.".format(field) @@ -323,6 +326,14 @@ def fillFields(csvfile, fields): tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], "")) if fields[20] > -1: tmpReq.setJobID(row[fields[20]]) + if fields[21] > -1: + tmpReq.setNotes(row[fields[21]]) + if fields[22] > -1: + tmpReq.setMcMTag(row[fields[22]].split(" ")) + if fields[23] > -1: + tmpReq.setSequencesBeamspot(row[fields[23]]) + if fields[24] > -1: + tmpReq.setSequencesMagField(row[fields[24]]) requests.append(tmpReq) return requests, num_requests From 425beb48adf31cddc853f3460945aa17976e7623 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 21 Jan 2016 19:48:29 +0100 Subject: [PATCH 21/64] Ignore lines in CSV file that are commented out --- manageRequests.py | 2 ++ testRequests.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/manageRequests.py b/manageRequests.py index f40d130..5256c63 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -239,6 +239,8 @@ def fillFields(csvfile, fields, campaign, PWG, notCreate_, McMTags): requests = [] # List containing request objects num_requests = 0 for row in csv.reader(csvfile): + if row[0].startswith("#"): + continue num_requests += 1 tmpReq = Request() if fields[0] > -1: diff --git a/testRequests.py b/testRequests.py index ce07079..9023fc0 100755 --- a/testRequests.py +++ b/testRequests.py @@ -279,6 +279,8 @@ def fillFields(csvfile, fields): requests = [] # List containing request objects num_requests = 0 for row in csv.reader(csvfile): + if row[0].startswith("#"): + continue num_requests += 1 tmpReq = Request() if fields[0] > -1: From a50f6fc0e2e776760fbe5174be55c4817b5d0fe2 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Sat, 23 Jan 2016 19:55:49 +0100 Subject: [PATCH 22/64] Add enough elements to fields list --- testRequests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testRequests.py b/testRequests.py index 9023fc0..dc3e336 100755 --- a/testRequests.py +++ b/testRequests.py @@ -157,7 +157,7 @@ def exitDuplicateField(file_in_,field_): def getFields(csvfile): # List of indices for each field in CSV file list = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1] + -1, -1, -1, -1, -1, -1, -1, -1] header = csv.reader(csvfile).next() for ind, field in enumerate(header): field = field.lower() From b8007fa0a0c0c740f36ec58b545ce4ef038f9fee Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 2 Feb 2016 18:59:37 +0100 Subject: [PATCH 23/64] Make magField lowercase to match field --- manageRequests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manageRequests.py b/manageRequests.py index 5256c63..23b5344 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -189,7 +189,7 @@ def getFields(csvfile_, file_in_): if list[22] > -1: exitDuplicateField(file_in_, "Sequences beamspot") list[22] = ind - elif field in ['sequences magField']: + elif field in ['sequences magfield']: if list[23] > -1: exitDuplicateField(file_in_, "Sequences magField") list[23] = ind From c9f35e461750dbe411440bc6af6ecb0e23ebcae4 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Mon, 15 Feb 2016 22:13:12 +0100 Subject: [PATCH 24/64] Add Database and add requests to it --- database/README.md | 7 +++++ database/addRequests.py | 61 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 database/README.md create mode 100755 database/addRequests.py diff --git a/database/README.md b/database/README.md new file mode 100644 index 0000000..c8a7572 --- /dev/null +++ b/database/README.md @@ -0,0 +1,7 @@ +Database +======== + +Schema +------ + +CREATE TABLE RequestSets(SetID INTEGER PRIMARY KEY, Process TEXT, RequesterID INTEGER, ContactID INTEGER, Tag INTEGER, Events INTEGER, Notes TEXT, RequestType INTEGER, GSNew INTEGER, GSValidating INTEGER, GSValidated INTEGER, GSDefined INTEGER, GSSubmitted INTEGER, GSDone INTEGER, FOREIGN KEY(ContactID) REFERENCES Contacts(ContactID), FOREIGN KEY(RequesterID) REFERENCES Requesters(RequesterID), FOREIGN KEY(RequestType) REFERENCES RequestTypes(RequestType)); diff --git a/database/addRequests.py b/database/addRequests.py new file mode 100755 index 0000000..4812959 --- /dev/null +++ b/database/addRequests.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +################################ +# +# addRequests.py +# +# Script to add requests to database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +import sys + + +def interrogate(): + loop = True + while loop: + process = raw_input("Process: ") + #requester = raw_input("Requester name: ") + #email = raw_input("Requester email: ") + tag = raw_input("Tag: ") + events = raw_input ("Events: ") + note = raw_input("Notes: ") + + yn = raw_input("Are these acceptable? [y/n] ") + while True: + if yn.lower() in ["y", "yes"]: + loop = False + break + elif yn.lower() in ["n", "no"]: + break + else: + yn = raw_input('Please enter "yes" or "no". ') + + return [process, tag, events, note] + + +def addRequestSet(answers): + conn = sqlite3.connect('EXO_MC_Requests.db') + c = conn.cursor() + c.execute('INSERT INTO RequestSets VALUES(NULL, "{0}", NULL, NULL, "{1}", "{2}", "{3}", 0, 0, 0, 0, 0, 0, 0);'.\ + format(answers[0], answers[1], answers[2], answers[3])) + conn.commit() + conn.close() + + print "Sucessfully added {0} {1} to RequestSets table.".\ + format(answers[0], answers[1]) + return + +def main(): + answers = interrogate() + addRequestSet(answers) + + return + + +if __name__ == '__main__': + main() From fa0c9a1151e2847bd776e2ba69ac2394cf2a7be0 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 17 Feb 2016 19:27:10 +0100 Subject: [PATCH 25/64] Fields for all campaigns --- database/addRequests.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/database/addRequests.py b/database/addRequests.py index 4812959..e053ee9 100755 --- a/database/addRequests.py +++ b/database/addRequests.py @@ -13,17 +13,21 @@ import sqlite3 import argparse import sys +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * def interrogate(): loop = True while loop: process = raw_input("Process: ") - #requester = raw_input("Requester name: ") - #email = raw_input("Requester email: ") + requester = raw_input("Requester name: ") + email = raw_input("Requester email: ") tag = raw_input("Tag: ") events = raw_input ("Events: ") note = raw_input("Notes: ") + spreadsheet = raw_input("Spreadsheet: ") + campaign = raw_input("Campaign: ") yn = raw_input("Are these acceptable? [y/n] ") while True: @@ -35,14 +39,18 @@ def interrogate(): else: yn = raw_input('Please enter "yes" or "no". ') - return [process, tag, events, note] + return [process, requester, email, tag, events, note, spreadsheet, campaign] def addRequestSet(answers): + mcm = restful(dev=False) + req_list = mcm.getA('requests', query='tags={0}&member_of_campaign={1}'.format( + answers[3], answers[7])) + multiplicity = len(req_list) conn = sqlite3.connect('EXO_MC_Requests.db') c = conn.cursor() - c.execute('INSERT INTO RequestSets VALUES(NULL, "{0}", NULL, NULL, "{1}", "{2}", "{3}", 0, 0, 0, 0, 0, 0, 0);'.\ - format(answers[0], answers[1], answers[2], answers[3])) + c.execute('INSERT INTO RequestSets (Process, Tag, Events, Notes, Spreadsheet, RequestMultiplicity) VALUES ("{0}", "{1}", {2}, "{3}", "{4}", {5});'.format( + answers[0], answers[3], answers[4], answers[5], answers[6], multiplicity)) conn.commit() conn.close() From 01272adeba144f8b560518d0f2aa4d7b215644c8 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 17 Feb 2016 19:41:39 +0100 Subject: [PATCH 26/64] Check status of requests in database --- database/check.py | 92 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100755 database/check.py diff --git a/database/check.py b/database/check.py new file mode 100755 index 0000000..0b6c31f --- /dev/null +++ b/database/check.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +################################ +# +# check.py +# +# Script to check status of requests and save in database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +import sys +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * +import time + + +def getRequestSets(): + mcm = restful(dev=False) + + wmLHE_campagin = "RunIIWinter15wmLHE" + pLHE_campaign = "RunIIWinter15pLHE" + GS_campaign = "RunIISummer15GS" + DR_campaign = "RunIIFall15DR76" + Mini_campaign = "RunIIFall15MiniAODv1" + Miniv2_campaign = "RunIIFall15MiniAODv2" + status_name = [["LHE_New", "LHE_Validating", "LHE_Validated", "LHE_Defined", + "LHE_Approved", "LHE_Submitted", "LHE_Done"], + ["GS_New", "GS_Validating", "GS_Validated", "GS_Defined", + "GS_Approved", "GS_Submitted", "GS_Done"], + ["DR_New", "DR_Validating", "DR_Validated", "DR_Defined", + "DR_Approved", "DR_Submitted", "DR_Done"], + ["MiniAOD_New", "MiniAOD_Validating", "MiniAOD_Validated", + "MiniAOD_Defined", "MiniAOD_Approved", "MiniAOD_Submitted", + "MiniAOD_Done"], + ["MiniAODv2_New", "MiniAODv2_Validating", + "MiniAODv2_Validated", "MiniAODv2_Defined", + "MiniAODv2_Approved", "MiniAODv2_Submitted", + "MiniAODv2_Done"]] + + conn = sqlite3.connect('EXO_MC_Requests.db') + c = conn.cursor() + c.execute('SELECT SetID, Tag FROM RequestSets') + out = c.fetchall() + + print "Checking:" + for request in out: + print request[1] + + campaigns = ["", GS_campaign, DR_campaign, Mini_campaign, Miniv2_campaign] + req_list = mcm.getA('requests', query='tags={0}'.format( + request[1])) + statuses = [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0]] + for req in req_list: + if req['member_of_campaign'] == GS_campaign: + if req['approval'] == "none" and req['status'] == "new": + statuses[1][0] += 1 + if req['approval'] == "validation" and req['status'] == "new": + statuses[1][1] += 1 + if req['approval'] == "validation" and req['status'] == "validation": + statuses[1][2] += 1 + if req['approval'] == "define" and req['status'] == "defined": + statuses[1][3] += 1 + if req['approval'] == "approve" and req['status'] == "approved": + statuses[1][4] += 1 + if req['approval'] == "submit" and req['status'] == "submitted": + statuses[1][5] += 1 + if req['approval'] == "submit" and req['status'] == "done": + statuses[1][6] += 1 + #print statuses + #print request[0] + for i in range(len(statuses)): + for j in range(len(statuses[0])): + c.execute('UPDATE RequestSets SET {0} = {1} WHERE SetID = {2}'.format( + status_name[i][j], statuses[i][j], request[0])) + time.sleep(0.5) + conn.commit() + conn.close() + + return + +def main(): + getRequestSets() + + return + + +if __name__ == '__main__': + main() From 3220bb783c56f2390ebb7175986d9c039dc4ad5b Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 17 Feb 2016 19:42:21 +0100 Subject: [PATCH 27/64] Ignore database files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9dc8ee7..1340512 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ \#*\# LSFJOB* *RunII*sh +*.db From e637fecc968ce6e050bcf4e97d531d65e00814bc Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 17 Feb 2016 20:48:58 +0100 Subject: [PATCH 28/64] Make web page from database --- database/makeHTML.py | 99 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100755 database/makeHTML.py diff --git a/database/makeHTML.py b/database/makeHTML.py new file mode 100755 index 0000000..49a00eb --- /dev/null +++ b/database/makeHTML.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +################################ +# +# check.py +# +# Script to check status of requests and save in database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +#from __future__ import print_function + + +def makeAnalyzerHTML(): + fout = open('analyzer.html', 'w') + status_name = [["LHE_New", "LHE_Validating", "LHE_Validated", "LHE_Defined", + "LHE_Approved", "LHE_Submitted", "LHE_Done"], + ["GS_New", "GS_Validating", "GS_Validated", "GS_Defined", + "GS_Approved", "GS_Submitted", "GS_Done"], + ["DR_New", "DR_Validating", "DR_Validated", "DR_Defined", + "DR_Approved", "DR_Submitted", "DR_Done"], + ["MiniAOD_New", "MiniAOD_Validating", "MiniAOD_Validated", + "MiniAOD_Defined", "MiniAOD_Approved", "MiniAOD_Submitted", + "MiniAOD_Done"], + ["MiniAODv2_New", "MiniAODv2_Validating", + "MiniAODv2_Validated", "MiniAODv2_Defined", + "MiniAODv2_Approved", "MiniAODv2_Submitted", + "MiniAODv2_Done"]] + campaigns = ["RunIIWinter15*LHE", "RunIISummer15GS", "RunIIFall15DR76", + "RunIIFall15MiniAODv1", "RunIIFall15MiniAODv2"] + + fout.write("""\ + + + + + Exotica MC Status for Analyzers + + + + + + + + + + + + + + + + + +""") + + conn = sqlite3.connect('EXO_MC_Requests.db') + c = conn.cursor() + c.execute('SELECT * FROM RequestSets;') + out = c.fetchall() + + print "Filling:" + for request in out: + print request[4] + fout.write("""\ + + +""".format(request[1])) + for i in range(len(status_name)): + fout.write(" \n") + fout.write("\n") + conn.close() + + fout.write("""\ +
ProcessLHEGSDRMiniAODMiniAODv2
{0}".format( + request[4], campaigns[i])) + for j in range(len(status_name[0])): + fout.write("{0} ".format(request[10 + j + 7*i])) + fout.write("
+ + +""") + fout.close() + + return + +def main(): + makeAnalyzerHTML() + + return + + +if __name__ == '__main__': + main() From a16d43b27c6b23123b3da6db3e3bed5e79fee35a Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 17 Feb 2016 21:04:06 +0100 Subject: [PATCH 29/64] Check all campaigns --- database/check.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/database/check.py b/database/check.py index 0b6c31f..6bef24d 100755 --- a/database/check.py +++ b/database/check.py @@ -55,21 +55,22 @@ def getRequestSets(): request[1])) statuses = [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0]] for req in req_list: - if req['member_of_campaign'] == GS_campaign: - if req['approval'] == "none" and req['status'] == "new": - statuses[1][0] += 1 - if req['approval'] == "validation" and req['status'] == "new": - statuses[1][1] += 1 - if req['approval'] == "validation" and req['status'] == "validation": - statuses[1][2] += 1 - if req['approval'] == "define" and req['status'] == "defined": - statuses[1][3] += 1 - if req['approval'] == "approve" and req['status'] == "approved": - statuses[1][4] += 1 - if req['approval'] == "submit" and req['status'] == "submitted": - statuses[1][5] += 1 - if req['approval'] == "submit" and req['status'] == "done": - statuses[1][6] += 1 + for i in range(len(campaigns)): + if req['member_of_campaign'] == campaigns[i]: + if req['approval'] == "none" and req['status'] == "new": + statuses[i][0] += 1 + elif req['approval'] == "validation" and req['status'] == "new": + statuses[i][1] += 1 + elif req['approval'] == "validation" and req['status'] == "validation": + statuses[i][2] += 1 + elif req['approval'] == "define" and req['status'] == "defined": + statuses[i][3] += 1 + elif req['approval'] == "approve" and req['status'] == "approved": + statuses[i][4] += 1 + elif req['approval'] == "submit" and req['status'] == "submitted": + statuses[i][5] += 1 + elif req['approval'] == "submit" and req['status'] == "done": + statuses[i][6] += 1 #print statuses #print request[0] for i in range(len(statuses)): From b94004dda419387442946abe2dbcee42c83f1dae Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 17 Feb 2016 21:09:26 +0100 Subject: [PATCH 30/64] Simplify display to only done requests --- database/makeHTML.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/database/makeHTML.py b/database/makeHTML.py index 49a00eb..4c20a6e 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -54,7 +54,7 @@ def makeAnalyzerHTML(): LHE GS DR - MiniAOD + MiniAODv1 MiniAODv2 """) @@ -72,11 +72,8 @@ def makeAnalyzerHTML(): {0} """.format(request[1])) for i in range(len(status_name)): - fout.write(" ".format( - request[4], campaigns[i])) - for j in range(len(status_name[0])): - fout.write("{0} ".format(request[10 + j + 7*i])) - fout.write("\n") + fout.write(" {2}/{3} done\n".format( + request[4], campaigns[i], request[16 + 7*i], request[9])) fout.write("\n") conn.close() From 878c3971785b1fe14848f668ad23feb5115d68f4 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 18 Feb 2016 00:17:33 +0100 Subject: [PATCH 31/64] Increase sleep time --- database/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/check.py b/database/check.py index 6bef24d..1f0a202 100755 --- a/database/check.py +++ b/database/check.py @@ -77,7 +77,7 @@ def getRequestSets(): for j in range(len(statuses[0])): c.execute('UPDATE RequestSets SET {0} = {1} WHERE SetID = {2}'.format( status_name[i][j], statuses[i][j], request[0])) - time.sleep(0.5) + time.sleep(1) conn.commit() conn.close() From b57dc8c99cc997f0b8c2fb70ff3938866a829a71 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 18 Feb 2016 17:21:39 +0100 Subject: [PATCH 32/64] Tag copies of old requests --- addTags.py | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100755 addTags.py diff --git a/addTags.py b/addTags.py new file mode 100755 index 0000000..47348bc --- /dev/null +++ b/addTags.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python + +################################ +# +# addTags.py +# +# Script to add tags to old requests. +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sys +import os.path +import argparse +import time +import mcmscripts_config +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * # Load class to access McM + + +def getArguments(): + parser = argparse.ArgumentParser( + description='Tag old requests.') + + # Command line flags + parser.add_argument('ids', metavar='PrepIDs', + help='List of PrepIDs to tag copies of.') + parser.add_argument('tag', metavar='Tag') + + args_ = parser.parse_args() + return args_ + + +def fillIDRange(pwg, campaign, first, last): + first = int(first) + last = int(last) + chains = [] + if first > last: + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) + sys.exit(1) + + for number in range(first, last+1): + chains.append("{0}-{1}-{2:05d}".format(pwg, campaign, number)) + return chains + + +def parseIDList(compactList): + splitList = compactList.split(',') + chains = [] + for subList in splitList: + splitSubList = subList.split('-') + if len(splitSubList) == 3: + chains.append(subList) + elif len(splitSubList) == 4: + chains += fillIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[3]) + elif len(splitSubList) == 6: + if splitSubList[0] != splitSubList[3]: + print "Error: PrepID range must be for the same PWG." + sys.exit(1) + if splitSubList[1] != splitSubList[4]: + print "Error: PrepID range must be for the same chained campaign." + sys.exit(1) + chains += fillIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[5]) + else: + print "Error: Poorly formed PrepID list." + sys.exit(1) + return chains + + +def modifyRequest(mcm, req, tag): + for existing in req['tags']: + if existing.startswith("EXO"): + print " \033[0;33m{0} has already tagged with {1}\033[0;m".format(req['prepid'], existing) + return + req['tags'] += [tag] + answer = mcm.updateA('requests', req) # Update request + if answer['results']: + print " \033[0;32m{0} tagged {1}\033[0;m".format(req['prepid'], tag) + else: + print " \033[0;32m{0} not tagged\033[0;m".format(req['prepid']) + return + + +def tagRequests(prepids, tag): + mcm = restful(dev=False) + campaigns = ['RunIISummer15GS', 'RunIIFall15DR76', 'RunIIFall15MiniAODv1', + 'RunIIFall15MiniAODv2'] + + for prepid in prepids: + time.sleep(1.0) + req = mcm.getA('requests', prepid) + print "{0} {1}".format(prepid, req['dataset_name']) + if req['member_of_campaign'] == "RunIIWinter15wmLHE"\ + or req['member_of_campaign'] == "RunIIWinter15pLHE": + modifyRequest(mcm, req, tag) + dataset_name = req['dataset_name'] + for campaign in campaigns: + query_string = "dataset_name={0}&member_of_campaign={1}".format( + dataset_name, campaign) + failed_to_get = True + for tries in range(3): + time.sleep(0.5) + req_list = mcm.getA('requests', query=query_string) + if req_list is not None: + failed_to_get = False + break + if failed_to_get: + print " \033[0;31mCould not find {0} in {1}\033[0;m".format( + dataset_name, campaign) + continue + if len(req_list) > 1: + print " \033[0;31m{0} has too many requests in {1}\033[0;m".format( + dataset_name, campaign) + continue + if len(req_list) == 0: + print " \033[0;31m{0} does not exist in {1}\033[0;m".format( + dataset_name, campaign) + continue + req = req_list[0] + modifyRequest(mcm, req, tag) + return + + +def main(): + args = getArguments() + prepids = parseIDList(args.ids) + print "Tagging {0} base requests".format(len(prepids)) + tagRequests(prepids, args.tag) + + +if __name__ == '__main__': + main() From 86caafbdabbfa56d717d40038309a403fc6c332a Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 18 Feb 2016 20:32:43 +0100 Subject: [PATCH 33/64] Show number of samples --- addTags.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/addTags.py b/addTags.py index 47348bc..06acb8a 100755 --- a/addTags.py +++ b/addTags.py @@ -74,7 +74,7 @@ def parseIDList(compactList): def modifyRequest(mcm, req, tag): for existing in req['tags']: if existing.startswith("EXO"): - print " \033[0;33m{0} has already tagged with {1}\033[0;m".format(req['prepid'], existing) + print " \033[0;33m{0} has already been tagged with {1}\033[0;m".format(req['prepid'], existing) return req['tags'] += [tag] answer = mcm.updateA('requests', req) # Update request @@ -90,10 +90,14 @@ def tagRequests(prepids, tag): campaigns = ['RunIISummer15GS', 'RunIIFall15DR76', 'RunIIFall15MiniAODv1', 'RunIIFall15MiniAODv2'] + num = len(prepids) + counter = 0 + for prepid in prepids: + counter += 1 time.sleep(1.0) req = mcm.getA('requests', prepid) - print "{0} {1}".format(prepid, req['dataset_name']) + print "{0}/{1} {2} {3}".format(counter, num, prepid, req['dataset_name']) if req['member_of_campaign'] == "RunIIWinter15wmLHE"\ or req['member_of_campaign'] == "RunIIWinter15pLHE": modifyRequest(mcm, req, tag) From 1115997341c6c4ed2402d9965370ebef72f35c8d Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 18 Feb 2016 20:33:58 +0100 Subject: [PATCH 34/64] Add requests from a CSV file --- database/batchAddRequests.py | 55 ++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100755 database/batchAddRequests.py diff --git a/database/batchAddRequests.py b/database/batchAddRequests.py new file mode 100755 index 0000000..c69aa8e --- /dev/null +++ b/database/batchAddRequests.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +################################ +# +# batchAddRequests.py +# +# Script to add requests to database with a CSV file +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +import sys +import csv +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * +import time + + +def addRequestSet(answers, mcm): + req_list = mcm.getA('requests', query='tags={0}&member_of_campaign=RunIISummer15GS'.format( + answers[1])) + multiplicity = len(req_list) + events = 0 + for req in req_list: + events += req['total_events'] + conn = sqlite3.connect('EXO_MC_Requests.db') + c = conn.cursor() + c.execute('INSERT INTO RequestSets (Process, Tag, Events, Notes, Spreadsheet, RequestMultiplicity) VALUES ("{0}", "{1}", {2}, "{3}", "{4}", {5});'.format( + answers[0], answers[1], events, answers[6], answers[5], multiplicity)) + conn.commit() + conn.close() + + print "Sucessfully added {0} {1} to RequestSets table.".\ + format(answers[0], answers[1]) + return + +def main(): + csvfile = open('fill.csv', 'r') + mcm = restful(dev=False) + print "Adding:" + for row in csv.reader(csvfile): + if row[0].startswith("#"): + continue + print row[0] + addRequestSet(row, mcm) + time.sleep(1) + + return + + +if __name__ == '__main__': + main() From 3852370426ab73ada33afccd3445f694c3f7add6 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Fri, 19 Feb 2016 19:55:16 +0100 Subject: [PATCH 35/64] Fix wmLHE campaign typo --- database/check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/check.py b/database/check.py index 1f0a202..6937b2a 100755 --- a/database/check.py +++ b/database/check.py @@ -21,7 +21,7 @@ def getRequestSets(): mcm = restful(dev=False) - wmLHE_campagin = "RunIIWinter15wmLHE" + wmLHE_campaign = "RunIIWinter15wmLHE" pLHE_campaign = "RunIIWinter15pLHE" GS_campaign = "RunIISummer15GS" DR_campaign = "RunIIFall15DR76" @@ -50,7 +50,7 @@ def getRequestSets(): for request in out: print request[1] - campaigns = ["", GS_campaign, DR_campaign, Mini_campaign, Miniv2_campaign] + campaigns = [wmLHE_campaign, GS_campaign, DR_campaign, Mini_campaign, Miniv2_campaign] req_list = mcm.getA('requests', query='tags={0}'.format( request[1])) statuses = [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0]] From 15cf1506cb5fc6e852957e1d29fee3d2b51dd542 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Fri, 19 Feb 2016 22:01:43 +0100 Subject: [PATCH 36/64] Add contact to database --- database/addContact.py | 57 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100755 database/addContact.py diff --git a/database/addContact.py b/database/addContact.py new file mode 100755 index 0000000..9332646 --- /dev/null +++ b/database/addContact.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +################################ +# +# addContact.py +# +# Script to add contact to database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 + + +def interrogate(): + loop = True + while loop: + name = raw_input("Name: ") + email = raw_input("email: ") + display_name = raw_input("Display name: ") + user_name = raw_input("Username: ") + + yn = raw_input("Are these acceptable? [y/n] ") + while True: + if yn.lower() in ["y", "yes"]: + loop = False + break + elif yn.lower() in ["n", "no"]: + break + else: + yn = raw_input('Please enter "yes" or "no". ') + + return [name, email, display_name, user_name] + + +def addContact(answers): + conn = sqlite3.connect('EXO_MC_Requests.db') + c = conn.cursor() + c.execute('INSERT INTO Contacts VALUES (Null, "{0}", "{1}", "{2}", "{3}");'.format( + answers[0], answers[1], answers[2], answers[3])) + conn.commit() + conn.close() + + print "Sucessfully added {0} to Contacts table.".format(answers[0]) + return + + +def main(): + answers = interrogate() + addContact(answers) + + return + + +if __name__ == '__main__': + main() From fdf12c227cb87bc44ce3c8cea26bb09c4df43cc2 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Sat, 20 Feb 2016 16:06:24 +0100 Subject: [PATCH 37/64] Add requester, get specific columns from table, add classes and wrapper to html --- database/makeHTML.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/database/makeHTML.py b/database/makeHTML.py index 4c20a6e..ff2edef 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -12,7 +12,6 @@ import sqlite3 import argparse -#from __future__ import print_function def makeAnalyzerHTML(): @@ -32,6 +31,7 @@ def makeAnalyzerHTML(): "MiniAODv2_Done"]] campaigns = ["RunIIWinter15*LHE", "RunIISummer15GS", "RunIIFall15DR76", "RunIIFall15MiniAODv1", "RunIIFall15MiniAODv2"] + campaign_classes = ["lhe", "gs", "dr", "miniaod", "miniaodv2"] fout.write("""\ +
+

Exotica MC

+ +

7_6_X Campaign

+

(RunIIWinter15wmLHE/RunIIWinter15pLHE →) RunIISummer15GS → RunIIFall15DR → RunIIFall15MiniAODv1 → RunIIFall15MiniAODv2

- - - - - - + + + + + + + """) conn = sqlite3.connect('EXO_MC_Requests.db') c = conn.cursor() - c.execute('SELECT * FROM RequestSets;') + c.execute('SELECT Process, Tag, RequestMultiplicity, LHE_Done, GS_Done, DR_Done, MiniAOD_Done, MiniAODv2_Done, RequesterID FROM RequestSets;') out = c.fetchall() - print "Filling:" for request in out: - print request[4] + c.execute('SELECT Name, Email FROM Requesters WHERE RequesterID = {0};'.format( + request[8])) + requester = c.fetchall() fout.write("""\ - -""".format(request[1])) + + +""".format(request[0], requester[0][1], requester[0][0])) for i in range(len(status_name)): - fout.write(" \n".format( - request[4], campaigns[i], request[16 + 7*i], request[9])) + fout.write(" \n".format( + campaign_classes[i], request[1], campaigns[i], request[3 + i], request[2])) fout.write("\n") conn.close() fout.write("""\
ProcessLHEGSDRMiniAODv1MiniAODv2ProcessRequesterLHEGSDRMiniAODv1MiniAODv2
{0}{0}{2}{2}/{3} done{3}/{4} done
+
+ """) fout.close() + print "Generated analyzer page" + return def main(): From 9757a2818a954a2d15ee6bba3463c534d735a6e7 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Sat, 20 Feb 2016 16:23:44 +0100 Subject: [PATCH 38/64] Move database location and html output to config file. --- database/addContact.py | 5 ++++- database/addRequests.py | 4 +++- database/batchAddRequests.py | 4 +++- database/check.py | 4 +++- database/makeHTML.py | 9 +++++++-- mcmscripts_config.py | 4 ++++ 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/database/addContact.py b/database/addContact.py index 9332646..ae05e88 100755 --- a/database/addContact.py +++ b/database/addContact.py @@ -11,6 +11,9 @@ ################################ import sqlite3 +import sys +sys.path.append('../') +import mcmscripts_config def interrogate(): @@ -35,7 +38,7 @@ def interrogate(): def addContact(answers): - conn = sqlite3.connect('EXO_MC_Requests.db') + conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() c.execute('INSERT INTO Contacts VALUES (Null, "{0}", "{1}", "{2}", "{3}");'.format( answers[0], answers[1], answers[2], answers[3])) diff --git a/database/addRequests.py b/database/addRequests.py index e053ee9..2f42abf 100755 --- a/database/addRequests.py +++ b/database/addRequests.py @@ -15,6 +15,8 @@ import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import * +sys.path.append('../') +import mcmscripts_config def interrogate(): @@ -47,7 +49,7 @@ def addRequestSet(answers): req_list = mcm.getA('requests', query='tags={0}&member_of_campaign={1}'.format( answers[3], answers[7])) multiplicity = len(req_list) - conn = sqlite3.connect('EXO_MC_Requests.db') + conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() c.execute('INSERT INTO RequestSets (Process, Tag, Events, Notes, Spreadsheet, RequestMultiplicity) VALUES ("{0}", "{1}", {2}, "{3}", "{4}", {5});'.format( answers[0], answers[3], answers[4], answers[5], answers[6], multiplicity)) diff --git a/database/batchAddRequests.py b/database/batchAddRequests.py index c69aa8e..025a3f8 100755 --- a/database/batchAddRequests.py +++ b/database/batchAddRequests.py @@ -17,6 +17,8 @@ sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import * import time +sys.path.append('../') +import mcmscripts_config def addRequestSet(answers, mcm): @@ -26,7 +28,7 @@ def addRequestSet(answers, mcm): events = 0 for req in req_list: events += req['total_events'] - conn = sqlite3.connect('EXO_MC_Requests.db') + conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() c.execute('INSERT INTO RequestSets (Process, Tag, Events, Notes, Spreadsheet, RequestMultiplicity) VALUES ("{0}", "{1}", {2}, "{3}", "{4}", {5});'.format( answers[0], answers[1], events, answers[6], answers[5], multiplicity)) diff --git a/database/check.py b/database/check.py index 6937b2a..f7e85ac 100755 --- a/database/check.py +++ b/database/check.py @@ -16,6 +16,8 @@ sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import * import time +sys.path.append('../') +import mcmscripts_config def getRequestSets(): @@ -41,7 +43,7 @@ def getRequestSets(): "MiniAODv2_Approved", "MiniAODv2_Submitted", "MiniAODv2_Done"]] - conn = sqlite3.connect('EXO_MC_Requests.db') + conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() c.execute('SELECT SetID, Tag FROM RequestSets') out = c.fetchall() diff --git a/database/makeHTML.py b/database/makeHTML.py index ff2edef..02885aa 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -12,10 +12,13 @@ import sqlite3 import argparse +import sys +sys.path.append('../') +import mcmscripts_config def makeAnalyzerHTML(): - fout = open('analyzer.html', 'w') + fout = open('{0}analyzer.html'.format(mcmscripts_config.html_location), 'w') status_name = [["LHE_New", "LHE_Validating", "LHE_Validated", "LHE_Defined", "LHE_Approved", "LHE_Submitted", "LHE_Done"], ["GS_New", "GS_Validating", "GS_Validated", "GS_Defined", @@ -65,7 +68,8 @@ def makeAnalyzerHTML(): """) - conn = sqlite3.connect('EXO_MC_Requests.db') + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() c.execute('SELECT Process, Tag, RequestMultiplicity, LHE_Done, GS_Done, DR_Done, MiniAOD_Done, MiniAODv2_Done, RequesterID FROM RequestSets;') out = c.fetchall() @@ -98,6 +102,7 @@ def makeAnalyzerHTML(): return + def main(): makeAnalyzerHTML() diff --git a/mcmscripts_config.py b/mcmscripts_config.py index 22824e4..2b1403c 100644 --- a/mcmscripts_config.py +++ b/mcmscripts_config.py @@ -8,3 +8,7 @@ # Automatically remove color when piping or redirecting output auto_bw = False + +# Database +database_location = 'MC_Requests.db' +html_location = '' From a9c6ae44ced7b83991399a08d5e42341b1db4b7a Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Sat, 20 Feb 2016 16:36:46 +0100 Subject: [PATCH 39/64] Add CSS --- database/global.css | 60 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 database/global.css diff --git a/database/global.css b/database/global.css new file mode 100644 index 0000000..8ac80a6 --- /dev/null +++ b/database/global.css @@ -0,0 +1,60 @@ +/************************* +** +** Global CSS for Exotica MC Status +** +** +*************************/ + +body { + background-color: #ffffff; + padding: 0; + margin: 0; + border: 0; + font-family: Helvetica, Ariel, sans-serif; +} + +div.wrapper { + margin: 20px 50px 20px 50px; +} + +a { + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +table { + border-collapse: collapse; +} + +tr:nth-child(even) { + background: #ffffff; +} + +tr:nth-child(odd) { + background: #dddddd; +} + +tr.table_header { + background-color: #999999; + border-bottom: 2px solid black; + text-color: #ffffff; +} + +th { + padding: 2px; +} + +td { + padding: 2px; +} + +th.lhe, th.gs, th.dr, th.miniaod, th.miniaodv2 { + width: 125px; +} + +td.lhe, td.gs, td.dr, td.miniaod, td.miniaodv2 { + text-align: center; +} From 36dca649eb2dc20701f3431ceea82f4c68041c2d Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Sat, 20 Feb 2016 20:37:21 +0100 Subject: [PATCH 40/64] Add HTML file for analyzer --- database/makeHTML.py | 110 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/database/makeHTML.py b/database/makeHTML.py index 02885aa..283d7e7 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -103,8 +103,118 @@ def makeAnalyzerHTML(): return +def makeContactHTML(): + fout = open('{0}contact.html'.format(mcmscripts_config.html_location), 'w') + status_name = [["LHE_New", "LHE_Validating", "LHE_Validated", "LHE_Defined", + "LHE_Approved", "LHE_Submitted", "LHE_Done"], + ["GS_New", "GS_Validating", "GS_Validated", "GS_Defined", + "GS_Approved", "GS_Submitted", "GS_Done"], + ["DR_New", "DR_Validating", "DR_Validated", "DR_Defined", + "DR_Approved", "DR_Submitted", "DR_Done"], + ["MiniAOD_New", "MiniAOD_Validating", "MiniAOD_Validated", + "MiniAOD_Defined", "MiniAOD_Approved", "MiniAOD_Submitted", + "MiniAOD_Done"], + ["MiniAODv2_New", "MiniAODv2_Validating", + "MiniAODv2_Validated", "MiniAODv2_Defined", + "MiniAODv2_Approved", "MiniAODv2_Submitted", + "MiniAODv2_Done"]] + campaigns = ["RunIIWinter15*LHE", "RunIISummer15GS", "RunIIFall15DR76", + "RunIIFall15MiniAODv1", "RunIIFall15MiniAODv2"] + campaign_classes = ["lhe", "gs", "dr", "miniaod", "miniaodv2"] + + fout.write("""\ + + + + + Exotica MC Status for Contacts + + + + + + + + +
+

Exotica MC

+ +

7_6_X Campaign

+

(RunIIWinter15wmLHE/RunIIWinter15pLHE →) RunIISummer15GS → RunIIFall15DR → RunIIFall15MiniAODv1 → RunIIFall15MiniAODv2

+ + + + + + + + + + + + + + + +""") + + conn = sqlite3.connect(mcmscripts_config.database_location) + + c = conn.cursor() + c.execute("""SELECT Process, RequesterID, ContactID, Tag, Events, Notes, Spreadsheet, RequestMultiplicity, + LHE_New, LHE_Validating, LHE_Validated, LHE_Defined, LHE_Approved, LHE_Submitted, LHE_Done, + GS_New, GS_Validating, GS_Validated, GS_Defined, GS_Approved, GS_Submitted, GS_Done, + DR_New, DR_Validating, DR_Validated, DR_Defined, DR_Approved, DR_Submitted, DR_Done, + MiniAOD_New, MiniAOD_Validating, MiniAOD_Validated, MiniAOD_Defined, MiniAOD_Approved, MiniAOD_Submitted, MiniAOD_Done, + MiniAODv2_New, MiniAODv2_Validating, MiniAODv2_Validated, MiniAODv2_Defined, MiniAODv2_Approved, MiniAODv2_Submitted, MiniAODv2_Done + FROM RequestSets;""") + out = c.fetchall() + + for request in out: + c.execute('SELECT Name, Email FROM Requesters WHERE RequesterID = {0};'.format( + request[1])) + requester = c.fetchall() + c.execute('SELECT DisplayName, Email FROM Contacts WHERE ContactID = {0};'.format( + request[2])) + contact = c.fetchall() + fout.write("""\ + + + + + + +""".format(request[0], request[3], requester[0][1], requester[0][0], contact[0][0], request[4])) + for i in range(len(status_name)): + fout.write(" \n".format( + campaign_classes[i], request[3], campaigns[i], + request[8 + i*7], request[9 + i*7], request[10 + i*7], + request[11 + i*7], request[12 + i*7], request[13 + i*7], + request[14 + i*7], request[7])) + fout.write("""\ + + +\n""".format(request[6], request[5])) + conn.close() + + fout.write("""\ +
ProcessTagRequesterContactEventsLHEGSDRMiniAODv1MiniAODv2SpreadsheetNotes
{0}{1}{3}{4}{5}{3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}		X{1}
+
+ + + +""") + fout.close() + + print "Generated contact page" + + return + + def main(): makeAnalyzerHTML() + makeContactHTML() return From c216ae4cf0408aa5dcac0a84270f5877676266d5 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 23 Feb 2016 20:13:40 +0100 Subject: [PATCH 41/64] Convert number of events to human readable format --- database/makeHTML.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/database/makeHTML.py b/database/makeHTML.py index 283d7e7..ef19f00 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -13,10 +13,21 @@ import sqlite3 import argparse import sys +import math sys.path.append('../') import mcmscripts_config +def display_number(n): + if n == 0: + return "0" + else: + prefix = ['', 'k', 'M'] + n = float(n) + prefix_id = max(0, min(2, int(math.floor(math.log10(abs(n))/3.0)))) + return "{0:.5g}{1}".format(n/10**(3*prefix_id), prefix[prefix_id]); + + def makeAnalyzerHTML(): fout = open('{0}analyzer.html'.format(mcmscripts_config.html_location), 'w') status_name = [["LHE_New", "LHE_Validating", "LHE_Validated", "LHE_Defined", @@ -84,7 +95,7 @@ def makeAnalyzerHTML(): {2} """.format(request[0], requester[0][1], requester[0][0])) for i in range(len(status_name)): - fout.write(" {3}/{4} done\n".format( + fout.write(" {3}/{4} done\n".format( campaign_classes[i], request[1], campaigns[i], request[3 + i], request[2])) fout.write("\n") conn.close() @@ -185,9 +196,10 @@ def makeContactHTML(): {3} {4} {5} -""".format(request[0], request[3], requester[0][1], requester[0][0], contact[0][0], request[4])) +""".format(request[0], request[3], requester[0][1], requester[0][0], contact[0][0], + display_number(request[4]))) for i in range(len(status_name)): - fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}\n".format( + fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}\n".format( campaign_classes[i], request[3], campaigns[i], request[8 + i*7], request[9 + i*7], request[10 + i*7], request[11 + i*7], request[12 + i*7], request[13 + i*7], From 2d2f502a378afe71b91b8fa467fc6684329a834a Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 23 Feb 2016 21:06:12 +0100 Subject: [PATCH 42/64] Only show spreadsheet link if present --- database/makeHTML.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/database/makeHTML.py b/database/makeHTML.py index ef19f00..198f418 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -204,10 +204,14 @@ def makeContactHTML(): request[8 + i*7], request[9 + i*7], request[10 + i*7], request[11 + i*7], request[12 + i*7], request[13 + i*7], request[14 + i*7], request[7])) + if request[6] == "": + fout.write("  \n") + else: + fout.write(" X".format( + request[6])) fout.write("""\ - X - {1} -\n""".format(request[6], request[5])) + {0} +\n""".format(request[5])) conn.close() fout.write("""\ From e336e4a1c31acc7c27450d6b8c301ea9945a13de Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 23 Feb 2016 23:00:15 +0100 Subject: [PATCH 43/64] Add arguments check only sets that are not done --- database/check.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/database/check.py b/database/check.py index f7e85ac..d5f5601 100755 --- a/database/check.py +++ b/database/check.py @@ -20,6 +20,20 @@ import mcmscripts_config +def getArguments(): + parser = argparse.ArgumentParser( + description='Check status of requests in McM and save in database.') + + # Command line flags + parser.add_argument('--check', action='store', dest='check', + help='Type of check to perform. 0 Check all sets, 1 Check sets that are not finished, 2 Check requests being validated.') + parser.add_argument('-t', '--tag', action='store', dest='tag', + help='Check status of requests based on tag.') + + args_ = parser.parse_args() + return args_ + + def getRequestSets(): mcm = restful(dev=False) @@ -45,7 +59,7 @@ def getRequestSets(): conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() - c.execute('SELECT SetID, Tag FROM RequestSets') + c.execute('SELECT SetID, Tag FROM RequestSets WHERE RequestMultiplicity != MiniAODv2_Done') out = c.fetchall() print "Checking:" @@ -86,6 +100,7 @@ def getRequestSets(): return def main(): + args = getArguments() # Setup flags and get arguments getRequestSets() return From 4c806823baae989173b7e399ef20476f4712e8e9 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 23 Feb 2016 23:33:18 +0100 Subject: [PATCH 44/64] Update schema in README file --- database/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/database/README.md b/database/README.md index c8a7572..977f1e1 100644 --- a/database/README.md +++ b/database/README.md @@ -4,4 +4,6 @@ Database Schema ------ -CREATE TABLE RequestSets(SetID INTEGER PRIMARY KEY, Process TEXT, RequesterID INTEGER, ContactID INTEGER, Tag INTEGER, Events INTEGER, Notes TEXT, RequestType INTEGER, GSNew INTEGER, GSValidating INTEGER, GSValidated INTEGER, GSDefined INTEGER, GSSubmitted INTEGER, GSDone INTEGER, FOREIGN KEY(ContactID) REFERENCES Contacts(ContactID), FOREIGN KEY(RequesterID) REFERENCES Requesters(RequesterID), FOREIGN KEY(RequestType) REFERENCES RequestTypes(RequestType)); +CREATE TABLE RequestSets(SetID INTEGER PRIMARY KEY, Process TEXT, RequesterID INTEGER, ContactID INTEGER, Tag INTEGER, Events INTEGER, Notes TEXT, Spreadsheet TEXT, RequestType INTEGER, RequestMultiplicity INTEGER, LHE_New INTEGER, LHE_Validating INTEGER, LHE_Validated INTEGER, LHE_Defined INTEGER, LHE_Approved INTEGER, LHE_Submitted INTEGER, LHE_Done INTEGER, GS_New INTEGER, GS_Validating INTEGER, GS_Validated INTEGER, GS_Defined INTEGER, GS_Approved INTEGER, GS_Submitted INTEGER, GS_Done INTEGER, DR_New INTEGER, DR_Validating INTEGER, DR_Validated INTEGER, DR_Defined INTEGER, DR_Approved INTEGER, DR_Submitted INTEGER, DR_Done INTEGER, MiniAOD_New INTEGER, MiniAOD_Validating INTEGER, MiniAOD_Validated INTEGER, MiniAOD_Defined INTEGER, MiniAOD_Approved INTEGER, MiniAOD_Submitted INTEGER, MiniAOD_Done INTEGER, MiniAODv2_New INTEGER, MiniAODv2_Validating INTEGER, MiniAODv2_Validated INTEGER, MiniAODv2_Defined INTEGER, MiniAODv2_Approved INTEGER, MiniAODv2_Submitted INTEGER, MiniAODv2_Done INTEGER, FOREIGN KEY(ContactID) REFERENCES Contacts(ContactID), FOREIGN KEY(RequesterID) REFERENCES Requesters(RequesterID), FOREIGN KEY(RequestType) REFERENCES RequestTypes(RequestType)); +CREATE TABLE Contacts(ContactID INTEGER PRIMARY KEY, Name TEXT, Email TEXT, DisplayName TEXT, UserName TEXT); +CREATE TABLE Requesters(RequesterID INTEGER PRIMARY KEY, Name TEXT, Email TEXT); From a54ee19259f4229075f1d38b7b028cc9d320eb2c Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 24 Feb 2016 21:54:47 +0100 Subject: [PATCH 45/64] Add favicon --- database/favicon.ico | Bin 0 -> 1150 bytes database/makeHTML.py | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 database/favicon.ico diff --git a/database/favicon.ico b/database/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..4b21e0a55fd5ddef21764e38f2bf5eaa4e4b0f22 GIT binary patch literal 1150 zcmbW1$!}X#6vkiTCKTM*&8SAHAdRRfh$;vb34w|P5`qN<2?}f~0fI;^V%T(n)CCe< zfZ9bDU9doDs-_7kA#n!Vp>+nwvEwARgWb9roy1OD_c?>{5ZkYwe%w0ZA3#svIrp9W zzIVQJ&efHq67d{7C<)sro!uu%b&@0<6A&R0*^3J&Nn)=`y@%qvFelGhX!3uec+JsDrX@`A73Vrw9BKJTdDXWMCRBW%65?wDL;8pR$rCU7v{zHz;oT1gz zO=<;J=A*o%%W|k4p1uZc`X8{EKEXb5ifsG|NJx;B*xAr!s+&L?8EQjalB3@UQTjHOJ;jd!{X9s$a7&x=h^Z_=xB}c@%0qdojIzF1)j2G z@NONau&KiBI7x5sIlQy4itqRG&&;>+aA|?aW|SU(j1$9;sK1)wXq!TfsQufnZXD)! z2>FgcR&WHyOsBDpzbo|DFg!EI8+U><*g+c2v41j0#dw|rLiYz7yl;c{~RCq>`OP-bett!C5yBHe$qFzR+P9RkN{oH64Ra!5J&Q9`V8k0?&RH$apa$PUGoJY`7@xg~VS0`JnG|}* hCPv392Iq#DlU(#hu{Jo?8FD?K$FjhG_rU-4{{o-u07L)) literal 0 HcmV?d00001 diff --git a/database/makeHTML.py b/database/makeHTML.py index 198f418..5598f6f 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -56,6 +56,7 @@ def makeAnalyzerHTML(): Exotica MC Status for Analyzers + @@ -142,6 +143,7 @@ def makeContactHTML(): Exotica MC Status for Contacts + From 614c7e96d013bbeb1c55bc88575cfe0694c1cf5f Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 25 Feb 2016 20:30:41 +0100 Subject: [PATCH 46/64] Clone multiple requests to different campaign --- batchCloneRequests.py | 103 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 batchCloneRequests.py diff --git a/batchCloneRequests.py b/batchCloneRequests.py new file mode 100755 index 0000000..2741a2c --- /dev/null +++ b/batchCloneRequests.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +################################ +# +# batchCloneRequests.py +# +# Script to clone multiple McM requests. +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sys +import os.path +import argparse +import time +import mcmscripts_config +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * # Load class to access McM + + +def getArguments(): + parser = argparse.ArgumentParser( + description='Clone multiple McM requests.') + + # Command line flags + parser.add_argument('-i', '--ids', dest='ids', help= + 'List of PrepIDs to be cloned.') + parser.add_argument('-c', '--campaign', action='store', dest='campaign', + metavar='name', help='Set member_of_campaign.') + + args_ = parser.parse_args() + return args_ + + +def fillIDRange(pwg, campaign, first, last): + first = int(first) + last = int(last) + requests = [] + if first > last: + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) + sys.exit(1) + + for number in range(first, last+1): + requests.append("{0}-{1}-{2:05d}".format(pwg, campaign, number)) + return requests + + +def parseIDList(compactList): + print compactList + splitList = compactList.split(',') + requests = [] + for subList in splitList: + splitSubList = subList.split('-') + if len(splitSubList) == 3: + requests.append(subList) + elif len(splitSubList) == 4: + requests += fillIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[3]) + elif len(splitSubList) == 6: + if splitSubList[0] != splitSubList[3]: + print "Error: PrepID range must be for the same PWG." + sys.exit(1) + if splitSubList[1] != splitSubList[4]: + print "Error: PrepID range must be for the same chained campaign." + sys.exit(1) + requests += fillIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[5]) + else: + print "Error: Poorly formed PrepID list." + sys.exit(1) + return requests + + +def cloneRequests(requests, campaign): + # Create new requests be cloning an old one based on PrepId + useDev = False + mcm = restful(dev=useDev) # Get McM connection + + print "Adding {0} requests to McM using clone.".format(len(requests)) + for PrepID in requests: + clone_req = mcm.getA('requests', PrepID) # Get request to clone + clone_req['member_of_campaign'] = campaign + answer = mcm.clone(PrepID, clone_req) # Clone request + if answer['results']: + print "\033[0;32m{0} cloned from {1}\033[0;m".format( + answer['prepid'], PrepID) + else: + print "\033[0;31m{0} failed to be cloned to {1}\033[0;m".format( + PrepID, campaign) + time.sleep(0.5) + + +def main(): + args = getArguments() # Setup flags and get arguments + + requests = parseIDList(args.ids) + cloneRequests(requests, args.campaign) + + +if __name__ == '__main__': + main() From bb0e63ce3ce31a3e0374cb3034197569f176e5c9 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Mon, 14 Mar 2016 04:15:30 +0100 Subject: [PATCH 47/64] Add timestamp for most recent update --- database/global.css | 6 ++++++ database/makeHTML.py | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/database/global.css b/database/global.css index 8ac80a6..dd163ff 100644 --- a/database/global.css +++ b/database/global.css @@ -58,3 +58,9 @@ th.lhe, th.gs, th.dr, th.miniaod, th.miniaodv2 { td.lhe, td.gs, td.dr, td.miniaod, td.miniaodv2 { text-align: center; } + +p.update-time { + color: #999999; + font-size: 0.8em; + font-style: italic; +} diff --git a/database/makeHTML.py b/database/makeHTML.py index 5598f6f..6e992ef 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -14,6 +14,7 @@ import argparse import sys import math +import time sys.path.append('../') import mcmscripts_config @@ -103,11 +104,12 @@ def makeAnalyzerHTML(): fout.write("""\ +

Updated {0}

-""") +""".format(time.asctime())) fout.close() print "Generated analyzer page" @@ -218,11 +220,12 @@ def makeContactHTML(): fout.write("""\ +

Updated {0}

-""") +""".format(time.asctime())) fout.close() print "Generated contact page" From 8999477bf5525c85b0f96e97f4bc1f877c5c94fe Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Mon, 14 Mar 2016 04:29:12 +0100 Subject: [PATCH 48/64] Add key to status numbers --- database/makeHTML.py | 1 + 1 file changed, 1 insertion(+) diff --git a/database/makeHTML.py b/database/makeHTML.py index 6e992ef..311904d 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -157,6 +157,7 @@ def makeContactHTML():

7_6_X Campaign

(RunIIWinter15wmLHE/RunIIWinter15pLHE →) RunIISummer15GS → RunIIFall15DR → RunIIFall15MiniAODv1 → RunIIFall15MiniAODv2

+
Key:1
2
3
4
5
6
7/28		new
validating
validated
defined
approved
submitted
done/total
From df4556150d6b09dccef134310314a9196040dfbc Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Mon, 11 Apr 2016 02:10:31 +0200 Subject: [PATCH 49/64] Add script to make ticket --- makeTicket.py | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100755 makeTicket.py diff --git a/makeTicket.py b/makeTicket.py new file mode 100755 index 0000000..66707a9 --- /dev/null +++ b/makeTicket.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python + +################################ +# +# makeTicket.py +# +# Script to create tickets for MCCM. +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sys +import argparse +import pprint +import mcmscripts_config +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * # Load class to access McM + + +def getArguments(): + parser = argparse.ArgumentParser( + description='Script to creat tickets in McM.') + + # Command line flags + parser.add_argument('-i', '--ids', dest='ids', help='List of PrepIDs.') + parser.add_argument('-p', '--pwg', action='store', dest='pwg', + default=mcmscripts_config.pwg, + help='Set PWG. Defaults to %(default)s. Change default in config.py') + parser.add_argument('-c', '--chain', action='store', dest='chain', + metavar='name', help='Set chain.') + parser.add_argument('-b', '--block', action='store', dest='block', type=int, + default=3, help='Set block.') + parser.add_argument('-r', '--repititions', action='store', + dest='repititions', type=int, default=1, + help='Number of chains.') + parser.add_argument('-s', '--staged', dest='staged', action='store', + type=int, default=0, help='Number of events needed.') + + args_ = parser.parse_args() + return args_ + + +def interrogate(): + loop = True + while loop: + chain = raw_input("Chain: ") + block = raw_input("Block: ") + repititions = raw_input("Repititions: ") + staged = raw_input("Staged: ") + + yn = raw_input("Are these acceptable? [y/n] ") + while True: + if yn.lower() in ["y", "yes"]: + loop = False + break + elif yn.lower() in ["n", "no"]: + break + else: + yn = raw_input('Please enter "yes" or "no". ') + + return [chain, block, repititions, staged] + + +def formatIDRange(pwg, campaign, first, last): + first = int(first) + last = int(last) + if first > last: + print "Error: PrepID range out of order. {0}-{1}-{2:05d} > {0}-{1}-{3:05d}".format( + pwg, campaign, first, last) + sys.exit(3) + + return ['{0}-{1}-{2:05d}'.format(pwg, campaign, first), + '{0}-{1}-{2:05d}'.format(pwg, campaign, last)] + + +def parseIDList(compactList): + splitList = compactList.split(',') + requests = [] + for subList in splitList: + splitSubList = subList.split('-') + if len(splitSubList) == 3: + requests.append(subList) + elif len(splitSubList) == 4: + requests.append(formatIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[3])) + elif len(splitSubList) == 6: + if splitSubList[0] != splitSubList[3]: + print "Error: PrepID range must be for the same PWG." + sys.exit(2) + if splitSubList[1] != splitSubList[4]: + print "Error: PrepID range must be for the same campaign." + sys.exit(2) + requests.append(formatIDRange(splitSubList[0], splitSubList[1], + splitSubList[2], splitSubList[5])) + else: + print "Error: Poorly formed PrepID list." + sys.exit(1) + return requests + + +def createTicket(requests, pwg, chain, block, repititions, staged): + mcm = restful(dev=False) + ticket = {'prepid': pwg, 'pwg': pwg, 'block': block, + 'chains': [chain], 'repetitions': repititions, + 'requests': requests + } + + pprint.pprint(ticket) + + # ticket = {'prepid': 'EXO', 'pwg': 'EXO', 'block': 2, + # 'chains': ['RunIISpring16DR80PU2016MiniAODv1'], 'repetitions': 1, + # 'requests': [['EXO-RunIISummer15GS-00435', 'EXO-RunIISummer15GS-00509'], ['EXO-RunIISummer15GS-06132', 'EXO-RunIISummer15GS-06139']] + # } + # ticket = {'prepid': 'EXO', 'pwg': 'EXO', 'block': 2, + # 'chains': ['RunIISpring16DR80PU2016MiniAODv1wmLHE'], 'repetitions': 1, + # 'notes': 'For B2G', + # 'requests': ['EXO-RunIIWinter15wmLHE-00288'] + # } + + answer = mcm.putA('mccms', ticket) + if answer['results']: + print "Created {0}".format(answer['prepid']) + else: + print "Failed" + + return + + +def modifyTicket(): + mcm = restful(dev=False) + ticket = mcm.getA('mccms', 'EXO-2016Mar30-00024') + #ticket['block'] = 2 + #ticket['chains'] = ['RunIISpring16DR80PU2016MiniAODv1'] + #ticket['chains'] = ['RunIISpring16DR80PU2016MiniAODv1wmLHE'] + #ticket['repetitions'] = 1 + #ticket['requests'] = [['EXO-RunIIWinter15wmLHE-01572', + pprint.pprint(ticket) + + # answer = mcm.updateA('mccms', ticket) + # print answer + # if answer['results']: + # print "Modified" + # else: + # print "Failed" + + return + + +def main(): + args = getArguments() # Setup flags and get arguments + + (args.chain, args.block, args.repititions, args.staged) = interrogate() + + requests = parseIDList(args.ids) + print requests + + createTicket(requests, args.pwg, args.chain, args.block, args.repititions, + args.staged) + return + + +if __name__ == '__main__': + main() From eb8c3fbe373e523b234876b92fc2fdec832c1e48 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 26 Apr 2016 06:19:45 +0200 Subject: [PATCH 50/64] Create fragments for wmLHEGS requests --- manageRequests.py | 56 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/manageRequests.py b/manageRequests.py index 23b5344..067a237 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -16,6 +16,7 @@ import csv import pprint import time +import urllib2 import mcmscripts_config sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import * # Load class to access McM @@ -210,13 +211,13 @@ def formatFragment(file_, campaign_): # 13 TeV campaigns elif campaign_ in ['Fall13', 'RunIIFall14GS', 'RunIIWinter15GS', 'RunIIWinter15wmLHE', 'RunIIWinter15pLHE', - 'RunIISummer15GS']: + 'RunIISummer15GS', 'RunIISummer15wmLHEGS']: return "Configuration/GenProduction/python/ThirteenTeV/"+file_ else: print "Error: Cannot determine energy of campaign {0}.".format(campaign_) sys.exit(5) -def createLHEProducer(gridpack, cards): +def createLHEProducer(gridpack, cards, fragment, tag): code = """import FWCore.ParameterSet.Config as cms externalLHEProducer = cms.EDProducer("ExternalLHEProducer", @@ -233,6 +234,18 @@ def createLHEProducer(gridpack, cards): # Link to cards: # {0} """.format(cards) + + if fragment != "" and tag != "": + gen_fragment_url = "https://raw.githubusercontent.com/cms-sw/genproductions/{0}/{1}".format( + tag, fragment.split("Configuration/GenProduction/")[1]) + gen_fragment = urllib2.urlopen(gen_fragment_url).read() + code += """ +{0} + +# Link to generator fragment: +# {1} +""".format(gen_fragment, gen_fragment_url) + return code def fillFields(csvfile, fields, campaign, PWG, notCreate_, McMTags): @@ -298,10 +311,16 @@ def fillFields(csvfile, fields, campaign, PWG, notCreate_, McMTags): tmpReq.setProcessString(row[fields[17]]) if fields[18] > -1: if fields[19] > -1: - tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], - row[fields[19]])) + if fields[4] > -1 and fields[7]: + tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], + row[fields[19]], + formatFragment(row[fields[4]], campaign), + row[fields[7]])) + else: + tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], + row[fields[19]], "", "")) else: - tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], "")) + tmpReq.setMcMFrag(createLHEProducer(row[fields[18]], "", "", "")) if fields[20] > -1: tmpReq.setNotes(row[fields[20]]) if fields[21] > -1: @@ -338,12 +357,13 @@ def createRequests(requests, num_requests, doDryRun, useDev): new_req['dataset_name'] = reqFields.getDataSetName() if reqFields.useEvts(): new_req['total_events'] = reqFields.getEvts() - if reqFields.useFrag(): - new_req['name_of_fragment'] = reqFields.getFrag() - if reqFields.useTag(): - new_req['fragment_tag'] = reqFields.getTag() if reqFields.useMcMFrag(): new_req['fragment'] = reqFields.getMcMFrag() + else: + if reqFields.useFrag(): + new_req['name_of_fragment'] = reqFields.getFrag() + if reqFields.useTag(): + new_req['fragment_tag'] = reqFields.getTag() if reqFields.useTime(): new_req['time_event'] = reqFields.getTime() if reqFields.useSize(): @@ -456,12 +476,13 @@ def modifyRequests(requests, num_requests, doDryRun, useDev, isLHErequest): mod_req['mcdb_id'] = reqFields.getMCDBID() if reqFields.useEvts(): mod_req['total_events'] = reqFields.getEvts() - if reqFields.useFrag(): - mod_req['name_of_fragment'] = reqFields.getFrag() - if reqFields.useTag(): - mod_req['fragment_tag'] = reqFields.getTag() if reqFields.useMcMFrag(): mod_req['fragment'] = reqFields.getMcMFrag() + else: + if reqFields.useFrag(): + mod_req['name_of_fragment'] = reqFields.getFrag() + if reqFields.useTag(): + mod_req['fragment_tag'] = reqFields.getTag() if reqFields.useTime(): mod_req['time_event'] = reqFields.getTime() if reqFields.useSize(): @@ -550,12 +571,13 @@ def cloneRequests(requests, num_requests, doDryRun, useDev, cloneId_): clone_req['mcdb_id'] = reqFields.getMCDBID() if reqFields.useEvts(): clone_req['total_events'] = reqFields.getEvts() - if reqFields.useFrag(): - clone_req['name_of_fragment'] = reqFields.getFrag() - if reqFields.useTag(): - clone_req['fragment_tag'] = reqFields.getTag() if reqFields.useMcMFrag(): clone_req['fragment'] = reqFields.getMcMFrag() + else: + if reqFields.useFrag(): + clone_req['name_of_fragment'] = reqFields.getFrag() + if reqFields.useTag(): + clone_req['fragment_tag'] = reqFields.getTag() if reqFields.useTime(): clone_req['time_event'] = reqFields.getTime() if reqFields.useSize(): From fbe1366fd689c146bebab3e70a71cb5432322a0d Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Sun, 1 May 2016 22:12:54 +0200 Subject: [PATCH 51/64] New database model. Make addRequest.py compatible with new model. Create newInstance.py to creat additional instances of request sets. --- database/README.md | 75 +- database/addRequests.py | 94 +- database/database_model.svg | 8511 +++++++++++++++++++++++++++++++++++ database/newInstance.py | 128 + 4 files changed, 8792 insertions(+), 16 deletions(-) create mode 100644 database/database_model.svg create mode 100755 database/newInstance.py diff --git a/database/README.md b/database/README.md index 977f1e1..79c4d0a 100644 --- a/database/README.md +++ b/database/README.md @@ -1,9 +1,78 @@ Database ======== +This is a database to track sets of requests in McM. + Schema ------ -CREATE TABLE RequestSets(SetID INTEGER PRIMARY KEY, Process TEXT, RequesterID INTEGER, ContactID INTEGER, Tag INTEGER, Events INTEGER, Notes TEXT, Spreadsheet TEXT, RequestType INTEGER, RequestMultiplicity INTEGER, LHE_New INTEGER, LHE_Validating INTEGER, LHE_Validated INTEGER, LHE_Defined INTEGER, LHE_Approved INTEGER, LHE_Submitted INTEGER, LHE_Done INTEGER, GS_New INTEGER, GS_Validating INTEGER, GS_Validated INTEGER, GS_Defined INTEGER, GS_Approved INTEGER, GS_Submitted INTEGER, GS_Done INTEGER, DR_New INTEGER, DR_Validating INTEGER, DR_Validated INTEGER, DR_Defined INTEGER, DR_Approved INTEGER, DR_Submitted INTEGER, DR_Done INTEGER, MiniAOD_New INTEGER, MiniAOD_Validating INTEGER, MiniAOD_Validated INTEGER, MiniAOD_Defined INTEGER, MiniAOD_Approved INTEGER, MiniAOD_Submitted INTEGER, MiniAOD_Done INTEGER, MiniAODv2_New INTEGER, MiniAODv2_Validating INTEGER, MiniAODv2_Validated INTEGER, MiniAODv2_Defined INTEGER, MiniAODv2_Approved INTEGER, MiniAODv2_Submitted INTEGER, MiniAODv2_Done INTEGER, FOREIGN KEY(ContactID) REFERENCES Contacts(ContactID), FOREIGN KEY(RequesterID) REFERENCES Requesters(RequesterID), FOREIGN KEY(RequestType) REFERENCES RequestTypes(RequestType)); -CREATE TABLE Contacts(ContactID INTEGER PRIMARY KEY, Name TEXT, Email TEXT, DisplayName TEXT, UserName TEXT); -CREATE TABLE Requesters(RequesterID INTEGER PRIMARY KEY, Name TEXT, Email TEXT); +''' +CREATE TABLE RequestSets( + SetID INTEGER PRIMARY KEY, + Process TEXT, + Tag TEXT, + Events INTEGER, + RequestMultiplicity INTEGER, + Notes TEXT, + Spreadsheet TEXT, + Ticket TEXT); +CREATE TABLE Contacts( + ContactID INTEGER PRIMARY KEY, + Name TEXT NOT NULL, + Email TEXT, + DisplayName TEXT NOT NULL, + UserName TEXT); +CREATE TABLE Requesters( + RequesterID INTEGER PRIMARY KEY, + Name TEXT NOT NULL, + Email TEXT); +CREATE TABLE Instances( + InstanceID INTEGER PRIMARY KEY, + SetID INTEGER, + CampaignChainID INTEGER, + ContactID INTEGER, + RequesterID INTEGER, + PriorityBlock INTEGER, + FOREIGN KEY(SetID) REFERENCES RequestSets(SetID), + FOREIGN KEY(CampaignChainID) REFERENCES CampaignChains(CampaignChainID), + FOREIGN KEY(ContactID) REFERENCES Contacts(ContactID), + FOREIGN KEY(RequesterID) REFERENCES Requesters(RequesterID)); +CREATE TABLE Requests( + RequestsID INTEGER PRIMARY KEY, + CampaignID INTEGER NOT NULL, + New INTEGER NOT NULL, + Validating INTEGER NOT NULL, + Validated INTEGER NOT NULL, + Defined INTEGER NOT NULL, + Approved INTEGER NOT NULL, + Submitted INTEGER NOT NULL, + FOREIGN KEY(CampaignID) REFERENCES Campaigns(CampaignID)); +CREATE TABLE Instance_Requests( + InstanceID INTEGER NOT NULL, + RequestsID INTEGER NOT NULL, + PRIMARY KEY(InstanceID, RequestsID) + FOREIGN KEY(InstanceID) REFERENCES Instances(InstanceID), + FOREIGN KEY(RequestsID) REFERENCES Requests(RequestsID)); +CREATE TABLE SuperCampaigns( + SuperCampaignID INTEGER PRIMARY KEY, + Name TEXT NOT NULL, + Active INTEGER NOT NULL); +CREATE TABLE CampaignChains( + CampaignChainID INTEGER PRIMARY KEY, + Name TEXT, + SuperCampaignID INTEGER NOT NULL, + FOREIGN KEY(SuperCampaignID) REFERENCES SuperCampaigns(SuperCampaignID)); +CREATE TABLE Campaigns( + CampaignID INTEGER PRIMARY KEY, + Name TEXT NOT NULL, + Level INTEGER NOT NULL); +CREATE TABLE CampaignChain_Campaign( + CampaignChainID INTEGER NOT NULL, + CampaignID INTEGER NOT NULL, + OrderInChain INTEGER NOT NULL, + PRIMARY KEY(CampaignChainID, CampaignID), + FOREIGN KEY(CampaignChainID) REFERENCES CampaignChains(CampaignChainID), + FOREIGN KEY(CampaignID) REFERENCES Campaigns(CampaignID)); +''' + +![database model](database_model.svg?raw=true "database model") diff --git a/database/addRequests.py b/database/addRequests.py index 2f42abf..adb1d88 100755 --- a/database/addRequests.py +++ b/database/addRequests.py @@ -19,7 +19,7 @@ import mcmscripts_config -def interrogate(): +def Interrogate(): loop = True while loop: process = raw_input("Process: ") @@ -27,9 +27,25 @@ def interrogate(): email = raw_input("Requester email: ") tag = raw_input("Tag: ") events = raw_input ("Events: ") + multiplicity = raw_input ("Number of requests: ") note = raw_input("Notes: ") spreadsheet = raw_input("Spreadsheet: ") - campaign = raw_input("Campaign: ") + ticket = raw_input("Ticket: ") + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT SuperCampaignID, Name FROM SuperCampaigns +WHERE Active > 0;""") + out = c.fetchall() + for row in out: + print "{0} {1}".format(row[0], row[1]) + super_campaign = raw_input("SuperCampaign: ") + c.execute("""SELECT CampaignChainID, Name FROM CampaignChains +WHERE SuperCampaignID = {0};""".format(super_campaign)) + out = c.fetchall() + for row in out: + print "{0} {1}".format(row[0], row[1]) + campaign_chain = raw_input("Chain: ") + conn.close() yn = raw_input("Are these acceptable? [y/n] ") while True: @@ -41,28 +57,80 @@ def interrogate(): else: yn = raw_input('Please enter "yes" or "no". ') - return [process, requester, email, tag, events, note, spreadsheet, campaign] + return [process, requester, email, tag, events, multiplicity, note, + spreadsheet, ticket, super_campaign, campaign_chain] -def addRequestSet(answers): - mcm = restful(dev=False) - req_list = mcm.getA('requests', query='tags={0}&member_of_campaign={1}'.format( - answers[3], answers[7])) - multiplicity = len(req_list) +def AddRequester(name_, email_): conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() - c.execute('INSERT INTO RequestSets (Process, Tag, Events, Notes, Spreadsheet, RequestMultiplicity) VALUES ("{0}", "{1}", {2}, "{3}", "{4}", {5});'.format( - answers[0], answers[3], answers[4], answers[5], answers[6], multiplicity)) + c.execute('INSERT INTO Requesters (Name, Email) VALUES ("{0}", "{1}");'.format( + name_, email_)) + requester_id = c.lastrowid + conn.commit() + conn.close() + + print "Added new requester: {0} ({1})".format(name_, email_) + + return requester_id + + +def GetRequester(name_, email_): + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT RequesterID, Name, Email FROM Requesters +WHERE Name = "{0}" OR Email = "{1}";""".format(name_, email_)) + out = c.fetchall() + conn.close() + requester_id = 0 + if len(out) == 0: + requester_id = AddRequester(name_, email_) + elif len(out) == 1: + requester_id = out[0][0] + else: + print "Found more than one requester. Choose the ID to use. 0 creates new requester." + for row in out: + print "{0} {1} {2}".format(row[0], row[1], row[2]) + requester_id = raw_input("User ID: ") + if requester_id == 0: + requester_id = AddRequester(name_, email_) + return requester_id + + +def AddRequestSet(answers_, requester_id_): + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""INSERT INTO RequestSets (Process, Tag, Events, +RequestMultiplicity, Notes, Spreadsheet, Ticket) VALUES ("{0}", "{1}", {2}, {3}, +"{4}", "{5}", "{6}");""".format(answers_[0], answers_[3], answers_[4], answers_[5], + answers_[6], answers_[7], answers_[8])) + set_id = c.lastrowid + c.execute("""INSERT INTO Instances (SetID, CampaignChainID, ContactID, +RequesterID, PriorityBlock) VALUES ({0}, {1}, {2}, {3}, {4});""".format( + set_id, answers_[10], 1, requester_id_, 3)) + instance_id = c.lastrowid + c.execute("""SELECT Campaigns.CampaignID FROM CampaignChains +INNER JOIN CampaignChain_Campaign USING(CampaignChainID) +INNER JOIN Campaigns USING(CampaignID) WHERE CampaignChainID = {0};""".format(answers_[10])) + out = c.fetchall() + for campaign in out: + c.execute("""INSERT INTO Requests (CampaignID, New, Validating, +Validated, Defined, Approved, Submitted) VALUES ({0}, 0, 0, 0, 0, 0, 0);""".format(campaign[0])) + requests_id = c.lastrowid + c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) +VALUES ({0}, {1});""".format(instance_id, requests_id)) conn.commit() conn.close() print "Sucessfully added {0} {1} to RequestSets table.".\ - format(answers[0], answers[1]) + format(answers_[0], answers_[1]) return + def main(): - answers = interrogate() - addRequestSet(answers) + answers = Interrogate() + requester_id = GetRequester(answers[1], answers[2]) + AddRequestSet(answers, requester_id) return diff --git a/database/database_model.svg b/database/database_model.svg new file mode 100644 index 0000000..0efb83d --- /dev/null +++ b/database/database_model.svg @@ -0,0 +1,8511 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/database/newInstance.py b/database/newInstance.py new file mode 100755 index 0000000..f563d0c --- /dev/null +++ b/database/newInstance.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python + +################################ +# +# newInstance.py +# +# Script to add new instance of requests to database +# +# author: David G. Sheffield (Rutgers) +# +################################ + +import sqlite3 +import argparse +import sys +sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') +from rest import * +sys.path.append('../') +import mcmscripts_config + + +def Interrogate(): + loop = True + while loop: + tag = raw_input("Tag: ") + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT SuperCampaignID, Name FROM SuperCampaigns +WHERE Active > 0;""") + out = c.fetchall() + for row in out: + print "{0} {1}".format(row[0], row[1]) + super_campaign = raw_input("SuperCampaign: ") + c.execute("""SELECT CampaignChainID, Name FROM CampaignChains +WHERE SuperCampaignID = {0};""".format(super_campaign)) + out = c.fetchall() + for row in out: + print "{0} {1}".format(row[0], row[1]) + campaign_chain = raw_input("Chain: ") + conn.close() + + yn = raw_input("Are these acceptable? [y/n] ") + while True: + if yn.lower() in ["y", "yes"]: + loop = False + break + elif yn.lower() in ["n", "no"]: + break + else: + yn = raw_input('Please enter "yes" or "no". ') + + return [tag, campaign_chain] + + +def AddRequester(name_, email_): + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute('INSERT INTO Requesters (Name, Email) VALUES ("{0}", "{1}");'.format( + name_, email_)) + requester_id = c.lastrowid + conn.commit() + conn.close() + + print "Added new requester: {0} ({1})".format(name_, email_) + + return requester_id + + +def GetRequester(name_, email_): + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT RequesterID, Name, Email FROM Requesters +WHERE Name = "{0}" OR Email = "{1}";""".format(name_, email_)) + out = c.fetchall() + conn.close() + requester_id = 0 + if len(out) == 0: + requester_id = AddRequester(name_, email_) + elif len(out) == 1: + requester_id = out[0][0] + else: + print "Found more than one requester. Choose the ID to use. 0 creates new requester." + for row in out: + print "{0} {1} {2}".format(row[0], row[1], row[2]) + requester_id = raw_input("User ID: ") + if requester_id == 0: + requester_id = AddRequester(name_, email_) + return requester_id + + +def AddRequestSet(answers_): + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT Instances.SetID, Instances.ContactID, +Instances.RequesterID, Instances.PriorityBlock FROM Instances +INNER JOIN RequestSets USING(SetID) WHERE Tag = "{0}";""".format(answers_[0])) + out = c.fetchall() + c.execute("""INSERT INTO Instances (SetID, CampaignChainID, ContactID, +RequesterID, PriorityBlock) VALUES ({0}, {1}, {2}, {3}, {4});""".format( + out[0][0], answers_[1], out[0][1], out[0][2], out[0][3])) + instance_id = c.lastrowid + c.execute("""SELECT Campaigns.CampaignID FROM CampaignChains +INNER JOIN CampaignChain_Campaign USING(CampaignChainID) +INNER JOIN Campaigns USING(CampaignID) WHERE CampaignChainID = {0};""".format(answers_[1])) + out = c.fetchall() + for campaign in out: + c.execute("""INSERT INTO Requests (CampaignID, New, Validating, +Validated, Defined, Approved, Submitted) VALUES ({0}, 0, 0, 0, 0, 0, 0);""".format(campaign[0])) + requests_id = c.lastrowid + c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) +VALUES ({0}, {1});""".format(instance_id, requests_id)) + conn.commit() + conn.close() + + print "Sucessfully added {0} {1} to RequestSets table.".\ + format(answers_[0], answers_[1]) + return + + +def main(): + answers = Interrogate() + AddRequestSet(answers) + + return + + +if __name__ == '__main__': + main() From 19251dd8f988820fb6006c5f0f7c8d8a55adf638 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Sun, 1 May 2016 22:39:11 +0200 Subject: [PATCH 52/64] Fix readme --- database/README.md | 8 +- database/database_model.svg | 9986 ++++++----------------------------- 2 files changed, 1480 insertions(+), 8514 deletions(-) diff --git a/database/README.md b/database/README.md index 79c4d0a..e8e6db1 100644 --- a/database/README.md +++ b/database/README.md @@ -3,10 +3,12 @@ Database This is a database to track sets of requests in McM. +[View database model](database_model.svg) + Schema ------ -''' +``` CREATE TABLE RequestSets( SetID INTEGER PRIMARY KEY, Process TEXT, @@ -73,6 +75,4 @@ CREATE TABLE CampaignChain_Campaign( PRIMARY KEY(CampaignChainID, CampaignID), FOREIGN KEY(CampaignChainID) REFERENCES CampaignChains(CampaignChainID), FOREIGN KEY(CampaignID) REFERENCES Campaigns(CampaignID)); -''' - -![database model](database_model.svg?raw=true "database model") +``` diff --git a/database/database_model.svg b/database/database_model.svg index 0efb83d..7b6e014 100644 --- a/database/database_model.svg +++ b/database/database_model.svg @@ -1,8511 +1,1477 @@ - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 6ed00f6dce67faa860dff33b561385b42a89f73e Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Sun, 19 Jun 2016 23:41:10 +0200 Subject: [PATCH 53/64] Remove --reprocess from getting cookies. Allow subdirectory for gen fragment. --- README.md | 4 ++-- manageRequests.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e8e5bad..71cddf7 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,11 @@ Get cookies Some scripts require getting a CERN SSO cookie before using them. They can be obtained for the production instance of McM with -`cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess` +`cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb` and for the dev/test instance with -`cern-get-sso-cookie -u https://cms-pdmv-dev.cern.ch/mcm/ -o ~/private/dev-cookie.txt --krb --reprocess` +`cern-get-sso-cookie -u https://cms-pdmv-dev.cern.ch/mcm/ -o ~/private/dev-cookie.txt --krb` Run these commands before setting up a CMSSW environment. If these to not work, try to run [getCookie.sh](getCookie.sh). diff --git a/manageRequests.py b/manageRequests.py index 067a237..aeda9f5 100755 --- a/manageRequests.py +++ b/manageRequests.py @@ -203,7 +203,7 @@ def getFields(csvfile_, file_in_): return list def formatFragment(file_, campaign_): - if len(file_.split("/")) > 1: + if len(file_.split("/")) > 2: return file_ # 8TeV campaign elif campaign_ in ['Summer12']: From 724518094dc7f7c5e0a4d98f116aeff7b88bab60 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Mon, 27 Jun 2016 19:08:57 +0200 Subject: [PATCH 54/64] Finish updating to new database model --- database/addRequests.py | 5 +- database/check.py | 53 ++++++++++- database/makeHTML.py | 139 ++++++++++++--------------- database/newInstance.py | 202 +++++++++++++++++++++++++--------------- 4 files changed, 243 insertions(+), 156 deletions(-) diff --git a/database/addRequests.py b/database/addRequests.py index adb1d88..787761d 100755 --- a/database/addRequests.py +++ b/database/addRequests.py @@ -46,6 +46,7 @@ def Interrogate(): print "{0} {1}".format(row[0], row[1]) campaign_chain = raw_input("Chain: ") conn.close() + contact_id = raw_input("Contact: ") yn = raw_input("Are these acceptable? [y/n] ") while True: @@ -58,7 +59,7 @@ def Interrogate(): yn = raw_input('Please enter "yes" or "no". ') return [process, requester, email, tag, events, multiplicity, note, - spreadsheet, ticket, super_campaign, campaign_chain] + spreadsheet, ticket, super_campaign, campaign_chain, contact_id] def AddRequester(name_, email_): @@ -107,7 +108,7 @@ def AddRequestSet(answers_, requester_id_): set_id = c.lastrowid c.execute("""INSERT INTO Instances (SetID, CampaignChainID, ContactID, RequesterID, PriorityBlock) VALUES ({0}, {1}, {2}, {3}, {4});""".format( - set_id, answers_[10], 1, requester_id_, 3)) + set_id, answers_[10], answers_[11], requester_id_, 3)) instance_id = c.lastrowid c.execute("""SELECT Campaigns.CampaignID FROM CampaignChains INNER JOIN CampaignChain_Campaign USING(CampaignChainID) diff --git a/database/check.py b/database/check.py index d5f5601..618cc6d 100755 --- a/database/check.py +++ b/database/check.py @@ -99,9 +99,60 @@ def getRequestSets(): return + +def checkRequests(): + mcm = restful(dev=False) + + conn = sqlite3.connect(mcmscripts_config.database_location) + c = conn.cursor() + c.execute("""SELECT DISTINCT Tag, RequestMultiplicity, Done, Campaigns.Name, +RequestsID FROM Requests INNER JOIN Campaigns USING(CampaignID) +INNER JOIN Instance_Requests USING(RequestsID) INNER JOIN Instances +USING(InstanceID) INNER JOIN RequestSets USING(SetID);""") + out = c.fetchall() + + status_names = ["New", "Validating", "Validated", "Defined", "Approved", + "Submitted", "Done"] + + for row in out: + if row[1] == row[2]: + continue + print "{0} {1}".format(row[0], row[3]) + req_list = mcm.getA('requests', + query='tags={0}&member_of_campaign={1}'.format( + row[0], row[3])) + time.sleep(1) + statuses = [0, 0, 0, 0, 0, 0, 0] + for req in req_list: + if req['approval'] == "none" and req['status'] == "new": + statuses[0] += 1 + elif req['approval'] == "validation" and req['status'] == "new": + statuses[1] += 1 + elif req['approval'] == "validation" and req['status'] == "validation": + statuses[2] += 1 + elif req['approval'] == "define" and req['status'] == "defined": + statuses[3] += 1 + elif req['approval'] == "approve" and req['status'] == "approved": + statuses[4] += 1 + elif req['approval'] == "submit" and req['status'] == "submitted": + statuses[5] += 1 + elif req['approval'] == "submit" and req['status'] == "done": + statuses[6] += 1 + + for i, status_name in enumerate(status_names): + c.execute("""UPDATE Requests SET {0} = {1} WHERE RequestsID = {2}""".format( + status_name, statuses[i], row[4])) + + conn.commit() + conn.close() + + return + + def main(): args = getArguments() # Setup flags and get arguments - getRequestSets() + #getRequestSets() + checkRequests() return diff --git a/database/makeHTML.py b/database/makeHTML.py index 311904d..63f09f2 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -31,21 +31,6 @@ def display_number(n): def makeAnalyzerHTML(): fout = open('{0}analyzer.html'.format(mcmscripts_config.html_location), 'w') - status_name = [["LHE_New", "LHE_Validating", "LHE_Validated", "LHE_Defined", - "LHE_Approved", "LHE_Submitted", "LHE_Done"], - ["GS_New", "GS_Validating", "GS_Validated", "GS_Defined", - "GS_Approved", "GS_Submitted", "GS_Done"], - ["DR_New", "DR_Validating", "DR_Validated", "DR_Defined", - "DR_Approved", "DR_Submitted", "DR_Done"], - ["MiniAOD_New", "MiniAOD_Validating", "MiniAOD_Validated", - "MiniAOD_Defined", "MiniAOD_Approved", "MiniAOD_Submitted", - "MiniAOD_Done"], - ["MiniAODv2_New", "MiniAODv2_Validating", - "MiniAODv2_Validated", "MiniAODv2_Defined", - "MiniAODv2_Approved", "MiniAODv2_Submitted", - "MiniAODv2_Done"]] - campaigns = ["RunIIWinter15*LHE", "RunIISummer15GS", "RunIIFall15DR76", - "RunIIFall15MiniAODv1", "RunIIFall15MiniAODv2"] campaign_classes = ["lhe", "gs", "dr", "miniaod", "miniaodv2"] fout.write("""\ @@ -77,30 +62,41 @@ def makeAnalyzerHTML(): GS DR MiniAODv1 - MiniAODv2 """) conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() - c.execute('SELECT Process, Tag, RequestMultiplicity, LHE_Done, GS_Done, DR_Done, MiniAOD_Done, MiniAODv2_Done, RequesterID FROM RequestSets;') - out = c.fetchall() - - for request in out: - c.execute('SELECT Name, Email FROM Requesters WHERE RequesterID = {0};'.format( - request[8])) - requester = c.fetchall() - fout.write("""\ + c.execute("""SELECT SetID, Process, Tag, Events, RequestMultiplicity, Notes, +Spreadsheet FROM RequestSets;""") + request_sets = c.fetchall() + for request_set in request_sets: + c.execute("""SELECT InstanceID, DisplayName, Requesters.Name, +Requesters.Email, CampaignChains.Name FROM Instances INNER JOIN Contacts +USING(ContactID) INNER JOIN Requesters USING(RequesterID) +INNER JOIN CampaignChains USING(CampaignChainID) WHERE SetID = {0};""".format(request_set[0])) + instances = c.fetchall() + for instance in instances: + c.execute("""SELECT Campaigns.Name, New, Validating, Validated, +Defined, Approved, Submitted, Done FROM Requests INNER JOIN Instance_Requests +USING(RequestsID) INNER JOIN Campaigns USING(CampaignID) +WHERE InstanceID = {0} ORDER BY Level""".format(instance[0])) + requests = c.fetchall() + fout.write("""\ -""".format(request[0], requester[0][1], requester[0][0])) - for i in range(len(status_name)): - fout.write(" \n".format( - campaign_classes[i], request[1], campaigns[i], request[3 + i], request[2])) +""".format(request_set[1], instance[3], instance[2])) + class_offset = 0 + if len(requests) < 4: + fout.write(" \n") + class_offset = 1 + for i in range(len(requests)): + fout.write(" \n".format( + campaign_classes[i + class_offset], request_set[2], + requests[i][0], requests[i][7], request_set[4])) fout.write("\n") - conn.close() fout.write("""\
Process
{0} {2}{3}/{4} done {3}/{4}
@@ -119,21 +115,6 @@ def makeAnalyzerHTML(): def makeContactHTML(): fout = open('{0}contact.html'.format(mcmscripts_config.html_location), 'w') - status_name = [["LHE_New", "LHE_Validating", "LHE_Validated", "LHE_Defined", - "LHE_Approved", "LHE_Submitted", "LHE_Done"], - ["GS_New", "GS_Validating", "GS_Validated", "GS_Defined", - "GS_Approved", "GS_Submitted", "GS_Done"], - ["DR_New", "DR_Validating", "DR_Validated", "DR_Defined", - "DR_Approved", "DR_Submitted", "DR_Done"], - ["MiniAOD_New", "MiniAOD_Validating", "MiniAOD_Validated", - "MiniAOD_Defined", "MiniAOD_Approved", "MiniAOD_Submitted", - "MiniAOD_Done"], - ["MiniAODv2_New", "MiniAODv2_Validating", - "MiniAODv2_Validated", "MiniAODv2_Defined", - "MiniAODv2_Approved", "MiniAODv2_Submitted", - "MiniAODv2_Done"]] - campaigns = ["RunIIWinter15*LHE", "RunIISummer15GS", "RunIIFall15DR76", - "RunIIFall15MiniAODv1", "RunIIFall15MiniAODv2"] campaign_classes = ["lhe", "gs", "dr", "miniaod", "miniaodv2"] fout.write("""\ @@ -169,7 +150,7 @@ def makeContactHTML(): GS DR MiniAODv1 - MiniAODv2 + Spreadsheet Notes @@ -178,46 +159,48 @@ def makeContactHTML(): conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() - c.execute("""SELECT Process, RequesterID, ContactID, Tag, Events, Notes, Spreadsheet, RequestMultiplicity, - LHE_New, LHE_Validating, LHE_Validated, LHE_Defined, LHE_Approved, LHE_Submitted, LHE_Done, - GS_New, GS_Validating, GS_Validated, GS_Defined, GS_Approved, GS_Submitted, GS_Done, - DR_New, DR_Validating, DR_Validated, DR_Defined, DR_Approved, DR_Submitted, DR_Done, - MiniAOD_New, MiniAOD_Validating, MiniAOD_Validated, MiniAOD_Defined, MiniAOD_Approved, MiniAOD_Submitted, MiniAOD_Done, - MiniAODv2_New, MiniAODv2_Validating, MiniAODv2_Validated, MiniAODv2_Defined, MiniAODv2_Approved, MiniAODv2_Submitted, MiniAODv2_Done - FROM RequestSets;""") - out = c.fetchall() - - for request in out: - c.execute('SELECT Name, Email FROM Requesters WHERE RequesterID = {0};'.format( - request[1])) - requester = c.fetchall() - c.execute('SELECT DisplayName, Email FROM Contacts WHERE ContactID = {0};'.format( - request[2])) - contact = c.fetchall() - fout.write("""\ + c.execute("""SELECT SetID, Process, Tag, Events, RequestMultiplicity, Notes, +Spreadsheet FROM RequestSets;""") + request_sets = c.fetchall() + for request_set in request_sets: + c.execute("""SELECT InstanceID, DisplayName, Requesters.Name, +Requesters.Email, CampaignChains.Name FROM Instances INNER JOIN Contacts +USING(ContactID) INNER JOIN Requesters USING(RequesterID) +INNER JOIN CampaignChains USING(CampaignChainID) WHERE SetID = {0};""".format(request_set[0])) + instances = c.fetchall() + for instance in instances: + c.execute("""SELECT Campaigns.Name, New, Validating, Validated, +Defined, Approved, Submitted, Done FROM Requests INNER JOIN Instance_Requests +USING(RequestsID) INNER JOIN Campaigns USING(CampaignID) +WHERE InstanceID = {0} ORDER BY Level""".format(instance[0])) + requests = c.fetchall() + fout.write("""\ {0} {1} {3} {4} {5} -""".format(request[0], request[3], requester[0][1], requester[0][0], contact[0][0], - display_number(request[4]))) - for i in range(len(status_name)): - fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}\n".format( - campaign_classes[i], request[3], campaigns[i], - request[8 + i*7], request[9 + i*7], request[10 + i*7], - request[11 + i*7], request[12 + i*7], request[13 + i*7], - request[14 + i*7], request[7])) - if request[6] == "": - fout.write("  \n") - else: - fout.write(" X".format( - request[6])) - fout.write("""\ +""".format(request_set[1], request_set[2], instance[3], instance[2], + instance[1], display_number(request_set[3]))) + class_offset = 0 + if len(requests) < 4: + fout.write("  \n") + class_offset = 1 + for i in range(len(requests)): + fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}\n".format( + campaign_classes[i + class_offset], request_set[2], + requests[i][0], requests[i][1], requests[i][2], + requests[i][3], requests[i][4], requests[i][5], + requests[i][6], requests[i][7], request_set[4])) + if request_set[6] == "": + fout.write("  \n") + else: + fout.write(" X".format( + request_set[6])) + fout.write("""\ {0} -\n""".format(request[5])) - conn.close() +\n""".format(request_set[5])) fout.write("""\ diff --git a/database/newInstance.py b/database/newInstance.py index f563d0c..ded3cd0 100755 --- a/database/newInstance.py +++ b/database/newInstance.py @@ -4,7 +4,7 @@ # # newInstance.py # -# Script to add new instance of requests to database +# Script to add new instance of set of requests to database # # author: David G. Sheffield (Rutgers) # @@ -19,26 +19,110 @@ import mcmscripts_config -def Interrogate(): +# def AddRequester(name_, email_): +# conn = sqlite3.connect(mcmscripts_config.database_location) +# c = conn.cursor() +# c.execute('INSERT INTO Requesters (Name, Email) VALUES ("{0}", "{1}");'.format( +# name_, email_)) +# requester_id = c.lastrowid +# conn.commit() +# conn.close() + +# print "Added new requester: {0} ({1})".format(name_, email_) + +# return requester_id + + +# def GetRequester(name_, email_): +# conn = sqlite3.connect(mcmscripts_config.database_location) +# c = conn.cursor() +# c.execute("""SELECT RequesterID, Name, Email FROM Requesters +# WHERE Name = "{0}" OR Email = "{1}";""".format(name_, email_)) +# out = c.fetchall() +# conn.close() +# requester_id = 0 +# if len(out) == 0: +# requester_id = AddRequester(name_, email_) +# elif len(out) == 1: +# requester_id = out[0][0] +# else: +# print "Found more than one requester. Choose the ID to use. 0 creates new requester." +# for row in out: +# print "{0} {1} {2}".format(row[0], row[1], row[2]) +# requester_id = raw_input("User ID: ") +# if requester_id == 0: +# requester_id = AddRequester(name_, email_) +# return requester_id + + +# def AddRequestSet(answers_, requester_id_): +# conn = sqlite3.connect(mcmscripts_config.database_location) +# c = conn.cursor() +# c.execute("""INSERT INTO RequestSets (Process, Tag, Events, +# RequestMultiplicity, Notes, Spreadsheet, Ticket) VALUES ("{0}", "{1}", {2}, {3}, +# "{4}", "{5}", "{6}");""".format(answers_[0], answers_[3], answers_[4], answers_[5], +# answers_[6], answers_[7], answers_[8])) +# set_id = c.lastrowid +# c.execute("""INSERT INTO Instances (SetID, CampaignChainID, ContactID, +# RequesterID, PriorityBlock) VALUES ({0}, {1}, {2}, {3}, {4});""".format( +# set_id, answers_[10], answers_[11], requester_id_, 3)) +# instance_id = c.lastrowid +# c.execute("""SELECT Campaigns.CampaignID FROM CampaignChains +# INNER JOIN CampaignChain_Campaign USING(CampaignChainID) +# INNER JOIN Campaigns USING(CampaignID) WHERE CampaignChainID = {0};""".format(answers_[10])) +# out = c.fetchall() +# for campaign in out: +# c.execute("""INSERT INTO Requests (CampaignID, New, Validating, +# Validated, Defined, Approved, Submitted) VALUES ({0}, 0, 0, 0, 0, 0, 0);""".format(campaign[0])) +# requests_id = c.lastrowid +# c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) +# VALUES ({0}, {1});""".format(instance_id, requests_id)) +# conn.commit() +# conn.close() + +# print "Sucessfully added {0} {1} to RequestSets table.".\ +# format(answers_[0], answers_[1]) +# return + + +def main(): loop = True while loop: tag = raw_input("Tag: ") conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() - c.execute("""SELECT SuperCampaignID, Name FROM SuperCampaigns -WHERE Active > 0;""") - out = c.fetchall() - for row in out: - print "{0} {1}".format(row[0], row[1]) - super_campaign = raw_input("SuperCampaign: ") - c.execute("""SELECT CampaignChainID, Name FROM CampaignChains -WHERE SuperCampaignID = {0};""".format(super_campaign)) + c.execute("""SELECT SetID, Process, Events, RequestMultiplicity, Notes, +Spreadsheet, Ticket, ContactID, DisplayName, RequesterID, Requesters.Name, +PriorityBlock, CampaignChains.Name, SuperCampaigns.Name, InstanceID, CampaignChainID +FROM RequestSets INNER JOIN Instances USING(SetID) INNER JOIN CampaignChains +USING(CampaignChainID) INNER JOIN SuperCampaigns USING(SuperCampaignID) +INNER JOIN Contacts USING(ContactID) INNER JOIN Requesters USING(RequesterID) +WHERE Tag = "{0}";""".format( + tag)) out = c.fetchall() - for row in out: - print "{0} {1}".format(row[0], row[1]) - campaign_chain = raw_input("Chain: ") + instance_list = [row[14] for row in out] + print instance_list + print "Process: {0}".format(out[0][1]) + print "Events: {0}".format(out[0][2]) + print "Request Multiplicity: {0}".format(out[0][3]) + if out[0][4] != "": + print "Notes: {0}".format(out[0][4]) + print "Spreadsheet: {0}".format(out[0][5]) + print "Ticket: {0}".format(out[0][6]) + for i, row in enumerate(out): + print "{0} ----------".format(i) #row[14]) + print "Contact: {1}".format(row[7], row[8]) + print "Requester: {1}".format(row[9], row[10]) + print "Block: ".format(row[11]) + print "SuperCampaign: {0}".format(row[13]) + print "Chain: {0}".format(row[12]) + print "------------" conn.close() + print "" + row_num = int(raw_input("Instance to copy: ")) + chainID = int(raw_input("Chain ID (7,11 GS / 9,13 wmLHE / 10,14 pLHE): ")) + yn = raw_input("Are these acceptable? [y/n] ") while True: if yn.lower() in ["y", "yes"]: @@ -49,78 +133,46 @@ def Interrogate(): else: yn = raw_input('Please enter "yes" or "no". ') - return [tag, campaign_chain] - - -def AddRequester(name_, email_): - conn = sqlite3.connect(mcmscripts_config.database_location) - c = conn.cursor() - c.execute('INSERT INTO Requesters (Name, Email) VALUES ("{0}", "{1}");'.format( - name_, email_)) - requester_id = c.lastrowid - conn.commit() - conn.close() - - print "Added new requester: {0} ({1})".format(name_, email_) - - return requester_id + print "\n===========\n" + SetID = out[row_num][0] + ContactID = out[row_num][7] + RequesterID = out[row_num][9] + PriorityBlock = out[row_num][11] + old_InstanceID = out[row_num][14] -def GetRequester(name_, email_): conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() - c.execute("""SELECT RequesterID, Name, Email FROM Requesters -WHERE Name = "{0}" OR Email = "{1}";""".format(name_, email_)) - out = c.fetchall() - conn.close() - requester_id = 0 - if len(out) == 0: - requester_id = AddRequester(name_, email_) - elif len(out) == 1: - requester_id = out[0][0] - else: - print "Found more than one requester. Choose the ID to use. 0 creates new requester." - for row in out: - print "{0} {1} {2}".format(row[0], row[1], row[2]) - requester_id = raw_input("User ID: ") - if requester_id == 0: - requester_id = AddRequester(name_, email_) - return requester_id - - -def AddRequestSet(answers_): - conn = sqlite3.connect(mcmscripts_config.database_location) - c = conn.cursor() - c.execute("""SELECT Instances.SetID, Instances.ContactID, -Instances.RequesterID, Instances.PriorityBlock FROM Instances -INNER JOIN RequestSets USING(SetID) WHERE Tag = "{0}";""".format(answers_[0])) - out = c.fetchall() c.execute("""INSERT INTO Instances (SetID, CampaignChainID, ContactID, RequesterID, PriorityBlock) VALUES ({0}, {1}, {2}, {3}, {4});""".format( - out[0][0], answers_[1], out[0][1], out[0][2], out[0][3])) - instance_id = c.lastrowid - c.execute("""SELECT Campaigns.CampaignID FROM CampaignChains -INNER JOIN CampaignChain_Campaign USING(CampaignChainID) -INNER JOIN Campaigns USING(CampaignID) WHERE CampaignChainID = {0};""".format(answers_[1])) + SetID, chainID, ContactID, RequesterID, PriorityBlock)) + new_InstanceID = c.lastrowid + print "Added new instance" + + c.execute("""SELECT CampaignID FROM CampaignChain_Campaign +WHERE CampaignChainID = {0};""".format(chainID)) out = c.fetchall() - for campaign in out: - c.execute("""INSERT INTO Requests (CampaignID, New, Validating, -Validated, Defined, Approved, Submitted) VALUES ({0}, 0, 0, 0, 0, 0, 0);""".format(campaign[0])) - requests_id = c.lastrowid - c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) -VALUES ({0}, {1});""".format(instance_id, requests_id)) + for row in out: + c.execute("""SELECT CampaignID, RequestsID FROM Requests +INNER JOIN Instance_Requests USING(RequestsID) WHERE InstanceID = {0} +AND CampaignID = {1};""".format(old_InstanceID, row[0])) + out_requests = c.fetchall() + if (len(out_requests) > 0): + c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) +VALUES ({0}, {1});""".format(new_InstanceID, out_requests[0][1])) + print "Added existing request to instance" + else: + c.execute("""INSERT INTO Requests (CampaignID, New, Validating, +Validated, Defined, Approved, Submitted) VALUES ({0}, 0, 0, 0, 0, 0, 0);""".format( + row[0])) + new_RequestsID = c.lastrowid + c.execute("""INSERT INTO Instance_Requests (InstanceID, RequestsID) +VALUES ({0}, {1});""".format(new_InstanceID, new_RequestsID)) + print "Added new request to instance" + conn.commit() conn.close() - print "Sucessfully added {0} {1} to RequestSets table.".\ - format(answers_[0], answers_[1]) - return - - -def main(): - answers = Interrogate() - AddRequestSet(answers) - return From 8b625d5f1ab46e303058684a7fb6d5ec7581e6a8 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Mon, 27 Jun 2016 20:32:23 +0200 Subject: [PATCH 55/64] Put separate SuperCampaigns in different tables of the status pages --- database/makeHTML.py | 247 ++++++++++++++++++++++++++----------------- 1 file changed, 148 insertions(+), 99 deletions(-) diff --git a/database/makeHTML.py b/database/makeHTML.py index 63f09f2..ff5eb65 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -29,6 +29,151 @@ def display_number(n): return "{0:.5g}{1}".format(n/10**(3*prefix_id), prefix[prefix_id]); +def writeRequests(page, fout, c, super_campaign, request_set, instance): + campaign_classes = ["lhe", "gs", "dr", "miniaod"] + + c.execute("""\ +SELECT Campaigns.Name, + New, + Validating, + Validated, + Defined, + Approved, + Submitted, + Done +FROM Requests +INNER JOIN Instance_Requests USING(RequestsID) +INNER JOIN Campaigns USING(CampaignID) +WHERE InstanceID = {0} +ORDER BY Level""".format(instance[0])) + requests = c.fetchall() + class_offset = 0 + if len(requests) < 4: + fout.write("  \n") + class_offset = 1 + for i in range(len(requests)): + if page == 0: + fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}\n".format( + campaign_classes[i + class_offset], request_set[2], + requests[i][0], requests[i][1], requests[i][2], + requests[i][3], requests[i][4], requests[i][5], + requests[i][6], requests[i][7], request_set[4])) + elif page == 1: + fout.write(" {3}/{4}\n".format( + campaign_classes[i + class_offset], request_set[2], + requests[i][0], requests[i][7], request_set[4])) + return + + +def writeInstances(page, fout, c, super_campaign, request_set): + c.execute("""\ +SELECT InstanceID, + DisplayName, + Requesters.Name, + Requesters.Email, + CampaignChains.Name +FROM Instances +INNER JOIN Contacts USING(ContactID) +INNER JOIN Requesters USING(RequesterID) +INNER JOIN CampaignChains USING(CampaignChainID) +WHERE SetID = {0} + AND SuperCampaignID = {1};""".format(request_set[0], super_campaign[0])) + instances = c.fetchall() + for instance in instances: + fout.write("\n") + if page == 0: + fout.write("""\ + {0} + {1} + {3} + {4} + {5} +""".format(request_set[1], request_set[2], instance[3], instance[2], + instance[1], display_number(request_set[3]))) + elif page == 1: + fout.write("""\ + {0} + {2} +""".format(request_set[1], instance[3], instance[2])) + writeRequests(page, fout, c, super_campaign, request_set, instance) + if page == 0: + if request_set[6] == "": + fout.write("  \n") + else: + fout.write(" X\n".format( + request_set[6])) + if request_set[5] == "": + fout.write("  \n") + else: + fout.write(" {0}\n".format(request_set[5])) + fout.write("\n") + return + + +def writeRequestSets(page, fout, c, super_campaign): + c.execute("""\ +SELECT DISTINCT SetID, + Process, + Tag, + Events, + RequestMultiplicity, + Notes, + Spreadsheet +FROM RequestSets +INNER JOIN Instances USING(SetID) +INNER JOIN CampaignChains USING(CampaignChainID) +WHERE SuperCampaignID = {0};""".format(super_campaign[0])) + request_sets = c.fetchall() + for request_set in request_sets: + writeInstances(page, fout, c, super_campaign, request_set) + return + + +def writeSuperCampaigns(page, fout, c): + c.execute("""\ +SELECT SuperCampaignID, + Name +FROM SuperCampaigns +ORDER BY Active;""") + super_campaigns = c.fetchall() + for super_campaign in super_campaigns: + fout.write("""\ +

{0}

+ + +""".format(super_campaign[1])) + if page == 0: + fout.write("""\ + + + + + + + + + + + +""") + elif page == 1: + fout.write("""\ + + + + + + +""") + fout.write("\n") + + writeRequestSets(page, fout, c, super_campaign) + + fout.write("
ProcessTagRequesterContactEventsLHEGSDRMiniAODv1SpreadsheetNotesProcessRequesterLHEGSDRMiniAODv1
\n") + + return + + def makeAnalyzerHTML(): fout = open('{0}analyzer.html'.format(mcmscripts_config.html_location), 'w') campaign_classes = ["lhe", "gs", "dr", "miniaod", "miniaodv2"] @@ -52,51 +197,12 @@ def makeAnalyzerHTML():

Exotica MC

-

7_6_X Campaign

-

(RunIIWinter15wmLHE/RunIIWinter15pLHE →) RunIISummer15GS → RunIIFall15DR → RunIIFall15MiniAODv1 → RunIIFall15MiniAODv2

- - - - - - - - - """) conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() - c.execute("""SELECT SetID, Process, Tag, Events, RequestMultiplicity, Notes, -Spreadsheet FROM RequestSets;""") - request_sets = c.fetchall() - for request_set in request_sets: - c.execute("""SELECT InstanceID, DisplayName, Requesters.Name, -Requesters.Email, CampaignChains.Name FROM Instances INNER JOIN Contacts -USING(ContactID) INNER JOIN Requesters USING(RequesterID) -INNER JOIN CampaignChains USING(CampaignChainID) WHERE SetID = {0};""".format(request_set[0])) - instances = c.fetchall() - for instance in instances: - c.execute("""SELECT Campaigns.Name, New, Validating, Validated, -Defined, Approved, Submitted, Done FROM Requests INNER JOIN Instance_Requests -USING(RequestsID) INNER JOIN Campaigns USING(CampaignID) -WHERE InstanceID = {0} ORDER BY Level""".format(instance[0])) - requests = c.fetchall() - fout.write("""\ - - - -""".format(request_set[1], instance[3], instance[2])) - class_offset = 0 - if len(requests) < 4: - fout.write(" \n") - class_offset = 1 - for i in range(len(requests)): - fout.write(" \n".format( - campaign_classes[i + class_offset], request_set[2], - requests[i][0], requests[i][7], request_set[4])) - fout.write("\n") + writeSuperCampaigns(1, fout, c) fout.write("""\
ProcessRequesterLHEGSDRMiniAODv1
{0}{2} {3}/{4}
@@ -136,71 +242,14 @@ def makeContactHTML():

Exotica MC

-

7_6_X Campaign

-

(RunIIWinter15wmLHE/RunIIWinter15pLHE →) RunIISummer15GS → RunIIFall15DR → RunIIFall15MiniAODv1 → RunIIFall15MiniAODv2

Key:1
2
3
4
5
6
7/28		new
validating
validated
defined
approved
submitted
done/total
- - - - - - - - - - - - - - - + """) conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() - c.execute("""SELECT SetID, Process, Tag, Events, RequestMultiplicity, Notes, -Spreadsheet FROM RequestSets;""") - request_sets = c.fetchall() - for request_set in request_sets: - c.execute("""SELECT InstanceID, DisplayName, Requesters.Name, -Requesters.Email, CampaignChains.Name FROM Instances INNER JOIN Contacts -USING(ContactID) INNER JOIN Requesters USING(RequesterID) -INNER JOIN CampaignChains USING(CampaignChainID) WHERE SetID = {0};""".format(request_set[0])) - instances = c.fetchall() - for instance in instances: - c.execute("""SELECT Campaigns.Name, New, Validating, Validated, -Defined, Approved, Submitted, Done FROM Requests INNER JOIN Instance_Requests -USING(RequestsID) INNER JOIN Campaigns USING(CampaignID) -WHERE InstanceID = {0} ORDER BY Level""".format(instance[0])) - requests = c.fetchall() - fout.write("""\ - - - - - - -""".format(request_set[1], request_set[2], instance[3], instance[2], - instance[1], display_number(request_set[3]))) - class_offset = 0 - if len(requests) < 4: - fout.write(" \n") - class_offset = 1 - for i in range(len(requests)): - fout.write(" \n".format( - campaign_classes[i + class_offset], request_set[2], - requests[i][0], requests[i][1], requests[i][2], - requests[i][3], requests[i][4], requests[i][5], - requests[i][6], requests[i][7], request_set[4])) - if request_set[6] == "": - fout.write(" \n") - else: - fout.write(" ".format( - request_set[6])) - fout.write("""\ - -\n""".format(request_set[5])) + writeSuperCampaigns(0, fout, c) fout.write("""\
ProcessTagRequesterContactEventsLHEGSDRMiniAODv1SpreadsheetNotes
{0}{1}{3}{4}{5} {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}		 X{0}
From e58fa1e74ab9136d025bd0910fee02e441302fa2 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Mon, 27 Jun 2016 21:11:47 +0200 Subject: [PATCH 56/64] Add Settings table to database. Save time of McM check and use in status pages. --- database/README.md | 4 ++++ database/check.py | 5 +++++ database/makeHTML.py | 15 ++++++++++++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/database/README.md b/database/README.md index e8e6db1..3f962d6 100644 --- a/database/README.md +++ b/database/README.md @@ -75,4 +75,8 @@ CREATE TABLE CampaignChain_Campaign( PRIMARY KEY(CampaignChainID, CampaignID), FOREIGN KEY(CampaignChainID) REFERENCES CampaignChains(CampaignChainID), FOREIGN KEY(CampaignID) REFERENCES Campaigns(CampaignID)); +CREATE TABLE Settings( + SettingID INTEGER PRIMARY KEY, + Value TEXT, + Description TEXT); ``` diff --git a/database/check.py b/database/check.py index 618cc6d..6d20f6b 100755 --- a/database/check.py +++ b/database/check.py @@ -143,6 +143,11 @@ def checkRequests(): c.execute("""UPDATE Requests SET {0} = {1} WHERE RequestsID = {2}""".format( status_name, statuses[i], row[4])) + c.execute("""\ +UPDATE Settings +SET Value = "{0}" +WHERE SettingID = 1;""".format(time.asctime())) + conn.commit() conn.close() diff --git a/database/makeHTML.py b/database/makeHTML.py index ff5eb65..5e27d63 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -14,7 +14,6 @@ import argparse import sys import math -import time sys.path.append('../') import mcmscripts_config @@ -204,6 +203,11 @@ def makeAnalyzerHTML(): c = conn.cursor() writeSuperCampaigns(1, fout, c) + c.execute("""\ +SELECT Value +FROM Settings +WHERE SettingID = 1""") + check_time = c.fetchone() fout.write("""\

Updated {0}

@@ -211,7 +215,7 @@ def makeAnalyzerHTML(): -""".format(time.asctime())) +""".format(check_time[0])) fout.close() print "Generated analyzer page" @@ -251,6 +255,11 @@ def makeContactHTML(): c = conn.cursor() writeSuperCampaigns(0, fout, c) + c.execute("""\ +SELECT Value +FROM Settings +WHERE SettingID = 1""") + check_time = c.fetchone() fout.write("""\

Updated {0}

@@ -258,7 +267,7 @@ def makeContactHTML(): -""".format(time.asctime())) +""".format(check_time[0])) fout.close() print "Generated contact page" From 8369c648e521bf68472fba08dbb63252bcab5017 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 28 Jun 2016 05:45:24 +0200 Subject: [PATCH 57/64] Add bar display to contact page --- database/makeHTML.py | 62 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/database/makeHTML.py b/database/makeHTML.py index 5e27d63..168b594 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -52,11 +52,55 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): class_offset = 1 for i in range(len(requests)): if page == 0: - fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}\n".format( + #ccccff,cc99ff,727272,98fb98,ff6666,ffc570,66aa66,ff4500 + fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}
".format( campaign_classes[i + class_offset], request_set[2], requests[i][0], requests[i][1], requests[i][2], requests[i][3], requests[i][4], requests[i][5], requests[i][6], requests[i][7], request_set[4])) + # + # fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}
".format( +# campaign_classes[i + class_offset], request_set[2], +# requests[i][0], requests[i][1], requests[i][2], +# requests[i][3], requests[i][4], requests[i][5], +# requests[i][6], requests[i][7], request_set[4])) +# chart_name = "{0}{1}".format(request_set[2], requests[i][0]).replace("-", "") + +# fout.write("""\ +# +# +# """.format(chart_name, requests[i][1], requests[i][2], requests[i][3], +# requests[i][4], requests[i][5], requests[i][6], requests[i][7], +# abs(requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4] + requests[i][5] + requests[i][6] + requests[i][7] - request_set[4]), +# request_set[2], requests[i][0])) + fout.write("\n") elif page == 1: fout.write(" {3}/{4}\n".format( campaign_classes[i + class_offset], request_set[2], @@ -246,10 +290,24 @@ def makeContactHTML():

Exotica MC

-
Key:1
2
3
4
5
6
7/28		new
validating
validated
defined
approved
submitted
done/total
+ + + + + + + + + +
 1new
 2validating
 3validated
Key:4defined
 5approved
 6submitted
 7/28done/total
""") + # + # + conn = sqlite3.connect(mcmscripts_config.database_location) c = conn.cursor() From e6cf16a962b185da6839e22227c199beb8087bb0 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 28 Jun 2016 07:12:21 +0200 Subject: [PATCH 58/64] User only bar display on both status pages. Add lists to title text and alt text of bar displays. Add spreadsheet icon from twiki. --- database/global.css | 9 +++++ database/makeHTML.py | 80 +++++++++++++++++++++++++++++++------------ database/table.gif | Bin 0 -> 136 bytes 3 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 database/table.gif diff --git a/database/global.css b/database/global.css index dd163ff..faf6017 100644 --- a/database/global.css +++ b/database/global.css @@ -64,3 +64,12 @@ p.update-time { font-size: 0.8em; font-style: italic; } + +img { + border: 0px; +} + +img.spreadsheet_icon { + width: 16px; + height: 16px; +} \ No newline at end of file diff --git a/database/makeHTML.py b/database/makeHTML.py index 168b594..afd1b7e 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -53,11 +53,22 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): for i in range(len(requests)): if page == 0: #ccccff,cc99ff,727272,98fb98,ff6666,ffc570,66aa66,ff4500 - fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}
".format( - campaign_classes[i + class_offset], request_set[2], - requests[i][0], requests[i][1], requests[i][2], - requests[i][3], requests[i][4], requests[i][5], - requests[i][6], requests[i][7], request_set[4])) + # fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}
\"{3}".format( + # campaign_classes[i + class_offset], request_set[2], + # requests[i][0], requests[i][1], requests[i][2], + # requests[i][3], requests[i][4], requests[i][5], + # requests[i][6], requests[i][7], request_set[4])) + fout.write(" \n".format(campaign_classes[i + class_offset])) + # fout.write(" {2}
{3}
{4}
{5}
{6}
{7}
{8}/{9}\n".format( + # request_set[2], requests[i][0], requests[i][1], + # requests[i][2], requests[i][3], requests[i][4], + # requests[i][5], requests[i][6], requests[i][7], + # request_set[4])) + fout.write(" \"{2}\n".format( + request_set[2], requests[i][0], requests[i][1], + requests[i][2], requests[i][3], requests[i][4], + requests[i][5], requests[i][6], requests[i][7], + request_set[4])) # # fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}
".format( # campaign_classes[i + class_offset], request_set[2], @@ -100,11 +111,18 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): # requests[i][4], requests[i][5], requests[i][6], requests[i][7], # abs(requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4] + requests[i][5] + requests[i][6] + requests[i][7] - request_set[4]), # request_set[2], requests[i][0])) - fout.write("\n") + fout.write(" \n") elif page == 1: - fout.write(" {3}/{4}\n".format( - campaign_classes[i + class_offset], request_set[2], - requests[i][0], requests[i][7], request_set[4])) + # fout.write(" {3}/{4}\n".format( + # campaign_classes[i + class_offset], request_set[2], + # requests[i][0], requests[i][7], request_set[4])) + fout.write(" \n".format(campaign_classes[i + class_offset])) + fout.write(" \"{2}\n".format( + request_set[2], requests[i][0], + requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4], + requests[i][5], requests[i][6], requests[i][7], + request_set[4])) + fout.write(" \n") return @@ -143,7 +161,17 @@ def writeInstances(page, fout, c, super_campaign, request_set): if request_set[6] == "": fout.write("  \n") else: - fout.write(" X\n".format( + fout.write(" \"X\"\n".format( + request_set[6])) + if request_set[5] == "": + fout.write("  \n") + else: + fout.write(" {0}\n".format(request_set[5])) + elif page == 1: + if request_set[6] == "": + fout.write("  \n") + else: + fout.write(" \"X\"\n".format( request_set[6])) if request_set[5] == "": fout.write("  \n") @@ -195,7 +223,7 @@ def writeSuperCampaigns(page, fout, c): LHE GS DR - MiniAODv1 + MiniAOD Spreadsheet Notes """) @@ -206,7 +234,9 @@ def writeSuperCampaigns(page, fout, c): LHE GS DR - MiniAODv1 + MiniAOD + Spreadsheet + Notes """) fout.write("\n") @@ -240,6 +270,13 @@ def makeAnalyzerHTML():

Exotica MC

+ + + + + +
Preparing requestsApproved to runRunningDone
+ """) conn = sqlite3.connect(mcmscripts_config.database_location) @@ -290,16 +327,15 @@ def makeContactHTML():

Exotica MC

- - - - - - - - - -
 1new
 2validating
 3validated
Key:4defined
 5approved
 6submitted
 7/28done/total
+ + + + + + + + +
NewValidatingValidatedDefinedApprovedSubmittedDone
""") diff --git a/database/table.gif b/database/table.gif new file mode 100644 index 0000000000000000000000000000000000000000..1f8c55b3bcefb8701dcadf089a54aa39221fbb7e GIT binary patch literal 136 zcmZ?wbhEHb6krfw*v!E2|NsBHckj-eIrG7T2j>_F109eOkX{BBTZ0pxt640vYquZ2 zyFJz^bDpZhqBT+sOlBuaeE2HX-OlGZvws%18#h~}!Q7&TFO2h#h`u%v>5U8POSSG~ HV6X-NwxcIU literal 0 HcmV?d00001 From 166fb2253f897aa04b84e5a4d93060d333ae7b26 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 28 Jun 2016 22:52:48 +0200 Subject: [PATCH 59/64] Use custom svg bar displays. Replace table.gif with sprite. --- database/global.css | 59 ++++++++++++++++-- database/makeHTML.py | 145 +++++++++++++++++++++++++------------------ database/sprite.png | Bin 0 -> 786 bytes database/table.gif | Bin 136 -> 0 bytes 4 files changed, 138 insertions(+), 66 deletions(-) create mode 100644 database/sprite.png delete mode 100644 database/table.gif diff --git a/database/global.css b/database/global.css index faf6017..ccc1e29 100644 --- a/database/global.css +++ b/database/global.css @@ -11,6 +11,7 @@ body { margin: 0; border: 0; font-family: Helvetica, Ariel, sans-serif; + font-size: 12pt; } div.wrapper { @@ -48,15 +49,15 @@ th { } td { - padding: 2px; + padding: 8px 2px 8px 2px; } th.lhe, th.gs, th.dr, th.miniaod, th.miniaodv2 { - width: 125px; + min-width: 100px; } td.lhe, td.gs, td.dr, td.miniaod, td.miniaodv2 { - text-align: center; + /*text-align: center;*/ } p.update-time { @@ -70,6 +71,54 @@ img { } img.spreadsheet_icon { - width: 16px; - height: 16px; + width: 21px; + height: 21px; +} + +svg.status { + margin-top: -18px; +} + +rect.new, path.new { + fill: #ccccff; + stroke-width: 0; +} + +rect.validating, path.validating { + fill: #cc99ff; + stroke-width: 0; +} + +rect.validated, path.validated { + fill: #6ba6e8; + stroke-width: 0; +} + +rect.defined, path.defined { + fill: #52fbc4; + stroke-width: 0; +} + +rect.approved, path.approved { + fill: #ffeba4; + stroke-width: 0; +} + +rect.submitted, path.submitted { + fill: #ffc570; + stroke-width: 0; +} + +rect.done, path.done { + fill: #66aa66; + stroke-width: 0; +} + +rect.unknown, path.unknown { + fill: #727272; + stroke-width: 0; +} + +g.tooltip text { + font-size: 13px; } \ No newline at end of file diff --git a/database/makeHTML.py b/database/makeHTML.py index afd1b7e..fe99405 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -14,9 +14,13 @@ import argparse import sys import math +import shutil sys.path.append('../') import mcmscripts_config +# Global variables +object_counter = 0 + def display_number(n): if n == 0: @@ -30,6 +34,7 @@ def display_number(n): def writeRequests(page, fout, c, super_campaign, request_set, instance): campaign_classes = ["lhe", "gs", "dr", "miniaod"] + global object_counter c.execute("""\ SELECT Campaigns.Name, @@ -52,72 +57,59 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): class_offset = 1 for i in range(len(requests)): if page == 0: - #ccccff,cc99ff,727272,98fb98,ff6666,ffc570,66aa66,ff4500 - # fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}
\"{3}".format( - # campaign_classes[i + class_offset], request_set[2], - # requests[i][0], requests[i][1], requests[i][2], - # requests[i][3], requests[i][4], requests[i][5], - # requests[i][6], requests[i][7], request_set[4])) fout.write(" \n".format(campaign_classes[i + class_offset])) - # fout.write(" {2}
{3}
{4}
{5}
{6}
{7}
{8}/{9}\n".format( - # request_set[2], requests[i][0], requests[i][1], - # requests[i][2], requests[i][3], requests[i][4], - # requests[i][5], requests[i][6], requests[i][7], - # request_set[4])) - fout.write(" \"{2}\n".format( - request_set[2], requests[i][0], requests[i][1], - requests[i][2], requests[i][3], requests[i][4], - requests[i][5], requests[i][6], requests[i][7], - request_set[4])) - # - # fout.write(" {3}
{4}
{5}
{6}
{7}
{8}
{9}/{10}
".format( -# campaign_classes[i + class_offset], request_set[2], -# requests[i][0], requests[i][1], requests[i][2], -# requests[i][3], requests[i][4], requests[i][5], -# requests[i][6], requests[i][7], request_set[4])) -# chart_name = "{0}{1}".format(request_set[2], requests[i][0]).replace("-", "") - -# fout.write("""\ -# -# -# """.format(chart_name, requests[i][1], requests[i][2], requests[i][3], -# requests[i][4], requests[i][5], requests[i][6], requests[i][7], -# abs(requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4] + requests[i][5] + requests[i][6] + requests[i][7] - request_set[4]), -# request_set[2], requests[i][0])) + fout.write("""\ + + + +""".format(request_set[2], requests[i][0])) + statuses = ["new", "validating", "validated", "defined", "approved", + "submitted", "done", "unkown"] + x_pos = 0.0 + tmp_object_counter = object_counter + for j in range(7): + bar_width = 0.0 + if request_set[4] != 0: + bar_width = float(requests[i][j+1])/float(request_set[4])*100.0 + fout.write("""\ + +""".format(x_pos, bar_width, statuses[j], tmp_object_counter)) + x_pos += bar_width + tmp_object_counter += 1 + fout.write("""\ + +""".format(x_pos, 100.0 - x_pos, tmp_object_counter)) + fout.write("""\ + + +""") + for j in range(7): + fout.write("""\ + + + + {2} {1} + +""".format(object_counter, statuses[j], requests[i][j+1])) + object_counter += 1 + fout.write("""\ + + + + {1} unkown + +""".format(object_counter, 0)) + object_counter += 1 + fout.write("""\ + +""") fout.write(" \n") elif page == 1: # fout.write(" {3}/{4}\n".format( # campaign_classes[i + class_offset], request_set[2], # requests[i][0], requests[i][7], request_set[4])) fout.write(" \n".format(campaign_classes[i + class_offset])) - fout.write(" \"{2}\n".format( + fout.write(" \"{2}\n".format( request_set[2], requests[i][0], requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4], requests[i][5], requests[i][6], requests[i][7], @@ -161,7 +153,7 @@ def writeInstances(page, fout, c, super_campaign, request_set): if request_set[6] == "": fout.write("  \n") else: - fout.write(" \"X\"\n".format( + fout.write(" \"X\"\n".format( request_set[6])) if request_set[5] == "": fout.write("  \n") @@ -171,7 +163,7 @@ def writeInstances(page, fout, c, super_campaign, request_set): if request_set[6] == "": fout.write("  \n") else: - fout.write(" \"X\"\n".format( + fout.write(" \"X\"\n".format( request_set[6])) if request_set[5] == "": fout.write("  \n") @@ -201,6 +193,8 @@ def writeRequestSets(page, fout, c, super_campaign): def writeSuperCampaigns(page, fout, c): + global object_counter + object_counter = 0 c.execute("""\ SELECT SuperCampaignID, Name @@ -320,6 +314,24 @@ def makeContactHTML(): + + + + @@ -369,9 +381,20 @@ def makeContactHTML(): return +def syncAuxiliaryFiles(): + shutil.copyfile("global.css", "{0}global.css".format(mcmscripts_config.html_location)) + shutil.copyfile("favicon.ico", "{0}favicon.ico".format(mcmscripts_config.html_location)) + shutil.copyfile("sprite.png", "{0}sprite.png".format(mcmscripts_config.html_location)) + + print "Copied auxiliary files" + + return + + def main(): makeAnalyzerHTML() makeContactHTML() + syncAuxiliaryFiles() return diff --git a/database/sprite.png b/database/sprite.png new file mode 100644 index 0000000000000000000000000000000000000000..b8521fa3a6525607235435502b0693a59b352bc2 GIT binary patch literal 786 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdzmUKs7M+SzC{oH>NSwWJ?9znhg z3{`3j3=J&|48MRv4KElNN(~qoUL`OvSj}Ky5HFasE6|34f$?f)NJL45ua8x7ey(0( zN`6wRUPW#JPzM8pO@$SZnVVXYs8ErclUHn2VXFjIVFhG^g!Ppaz)DK8ZIvQ?0~DO| zi&7O#^-S~(l1AfjnEKjFOT9D}DX)@^Za$W4-*MbbUihOG|wN zBYh(yU7!lx;>x^|#0uTKVr7USFmqf|i<65o3raHc^AtelCMM;Vme?vOfh>Xph&xL% z(-1c06+^uR^q@XSM&D4+Kp$>4P^%3{)XF(Ox1cDsxEN@@ouLg_C5jlr9*BDXqRdpF zOF`~|t49_?S09j(Uy=_DwW7>q&%BbN%5XaFUKJ)SO(ArYK!r*GstWWeJxca`jfsO-t-xyDO;U)St&pL1wK)RCpK*M3Ic-_92QGW@AhLU_Uw$s5z8TlXDW{`%wF zz5hcUN{ov&Cl@|_9{W@B`ptC?f(ODC`cF6?Dt&+XZk}U!qs`k_Hjl4t{G^!|zJQ}7 zy}?Fh&K|Wp2$_W#GPOQ$#6%hs|0`V<{%N}JdL2V+wdTgQyB5I2&A{O4>gTe~DWM4f DL`@LX literal 0 HcmV?d00001 diff --git a/database/table.gif b/database/table.gif deleted file mode 100644 index 1f8c55b3bcefb8701dcadf089a54aa39221fbb7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 136 zcmZ?wbhEHb6krfw*v!E2|NsBHckj-eIrG7T2j>_F109eOkX{BBTZ0pxt640vYquZ2 zyFJz^bDpZhqBT+sOlBuaeE2HX-OlGZvws%18#h~}!Q7&TFO2h#h`u%v>5U8POSSG~ HV6X-NwxcIU From 1f4f5be5a521d2ede2679008c0929f225dcc7f2a Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 29 Jun 2016 00:36:18 +0200 Subject: [PATCH 60/64] Switch to sprites --- database/global.css | 10 ++++++++++ database/makeHTML.py | 6 +++--- database/sprite.png | Bin 786 -> 0 bytes database/sprites.png | Bin 0 -> 936 bytes 4 files changed, 13 insertions(+), 3 deletions(-) delete mode 100644 database/sprite.png create mode 100644 database/sprites.png diff --git a/database/global.css b/database/global.css index ccc1e29..6e07daf 100644 --- a/database/global.css +++ b/database/global.css @@ -121,4 +121,14 @@ rect.unknown, path.unknown { g.tooltip text { font-size: 13px; +} + +div.spreadsheet_icon { + width: 32px; + height: 32px; + background: url(sprites.png) -5px -5px; +} + +td.spreadsheet a:hover { + text-decoration: none; } \ No newline at end of file diff --git a/database/makeHTML.py b/database/makeHTML.py index fe99405..a0606ef 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -153,7 +153,7 @@ def writeInstances(page, fout, c, super_campaign, request_set): if request_set[6] == "": fout.write("  \n") else: - fout.write(" \"X\"\n".format( + fout.write("
 
\n".format( request_set[6])) if request_set[5] == "": fout.write("  \n") @@ -163,7 +163,7 @@ def writeInstances(page, fout, c, super_campaign, request_set): if request_set[6] == "": fout.write("  \n") else: - fout.write(" \"X\"\n".format( + fout.write("
 
\n".format( request_set[6])) if request_set[5] == "": fout.write("  \n") @@ -384,7 +384,7 @@ def makeContactHTML(): def syncAuxiliaryFiles(): shutil.copyfile("global.css", "{0}global.css".format(mcmscripts_config.html_location)) shutil.copyfile("favicon.ico", "{0}favicon.ico".format(mcmscripts_config.html_location)) - shutil.copyfile("sprite.png", "{0}sprite.png".format(mcmscripts_config.html_location)) + shutil.copyfile("sprites.png", "{0}sprites.png".format(mcmscripts_config.html_location)) print "Copied auxiliary files" diff --git a/database/sprite.png b/database/sprite.png deleted file mode 100644 index b8521fa3a6525607235435502b0693a59b352bc2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 786 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdzmUKs7M+SzC{oH>NSwWJ?9znhg z3{`3j3=J&|48MRv4KElNN(~qoUL`OvSj}Ky5HFasE6|34f$?f)NJL45ua8x7ey(0( zN`6wRUPW#JPzM8pO@$SZnVVXYs8ErclUHn2VXFjIVFhG^g!Ppaz)DK8ZIvQ?0~DO| zi&7O#^-S~(l1AfjnEKjFOT9D}DX)@^Za$W4-*MbbUihOG|wN zBYh(yU7!lx;>x^|#0uTKVr7USFmqf|i<65o3raHc^AtelCMM;Vme?vOfh>Xph&xL% z(-1c06+^uR^q@XSM&D4+Kp$>4P^%3{)XF(Ox1cDsxEN@@ouLg_C5jlr9*BDXqRdpF zOF`~|t49_?S09j(Uy=_DwW7>q&%BbN%5XaFUKJ)SO(ArYK!r*GstWWeJxca`jfsO-t-xyDO;U)St&pL1wK)RCpK*M3Ic-_92QGW@AhLU_Uw$s5z8TlXDW{`%wF zz5hcUN{ov&Cl@|_9{W@B`ptC?f(ODC`cF6?Dt&+XZk}U!qs`k_Hjl4t{G^!|zJQ}7 zy}?Fh&K|Wp2$_W#GPOQ$#6%hs|0`V<{%N}JdL2V+wdTgQyB5I2&A{O4>gTe~DWM4f DL`@LX diff --git a/database/sprites.png b/database/sprites.png new file mode 100644 index 0000000000000000000000000000000000000000..afc1305691b422e45bb8eee07374608b592f5740 GIT binary patch literal 936 zcmV;Z16TZsP)XMo+$mW};px z1{ir#uaJ1SYb~AHm*aNtJs)@DU3cHnIeV|Y_CCM!Uu&Ix&LzZOPy!UxTXcp9G_VLF z&xZG?u1!S|Xkg_; zpb=;=0*;y209bEkD~M<|0B2_7MDz&26%n}&K#R=SlM1a=Dl1j>EDnTKRRCZ7Vx}|m zMl(A|L<i%1eyE%Rzm0A=QF0IDhY z10r(CpHFel9R*O&KNnT?Ab{^lKQnIuaG^fd%=W74$yPVA>w%WYp%Wr93&7_DigWG+ z5$#I@GP4t^y5G0WVCHcEQ~mi@5m^S{Su?0T2h_}7sA|2sCqUm2(cA{0W2!o@-sYTp zkqY~y`L2kp4hA%D-paBIQRTDdVCFjjeqMZ1*;RFM3f34A%_#_UL`2s1Bv1egnRzr7 zvYm$f7ZF(k;AJqNlc~y=$e{^`K@k~F7bPMZ@=pNOhN83;!1+{B+Nr9iV^K;GFtb;x zx)i{NjFi#uoO?<{^SxbFM7FmZ)AdCuD~G1~r!jM~)m!h|9*9VC-0s2t27r~`{$?&EawIR$=AV&UeFAB< z2hZA@H?Ahj{!i@Xa}(*%u#`e4(_>SI*B^X%*8>fdLw|x5k&ZqPMxYHE{U<;}#Yu=N zA8IY3{FG8?sL6XCXef|beu`B2qB*qFkj`3rA%$i+3)LTih9c?x6agA|>Wy^TO4(7*%&yNe194D<*4voA?BgvB}l0000< KMNUMnLSTX+v$DDX literal 0 HcmV?d00001 From 871e491ed04876332b92eedcaecb0c287095b5f5 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 29 Jun 2016 01:06:25 +0200 Subject: [PATCH 61/64] Use custom svg bar displays on both pages. Split unkown status into absent and extra. --- database/global.css | 7 ++- database/makeHTML.py | 104 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 92 insertions(+), 19 deletions(-) diff --git a/database/global.css b/database/global.css index 6e07daf..3d7cdab 100644 --- a/database/global.css +++ b/database/global.css @@ -114,11 +114,16 @@ rect.done, path.done { stroke-width: 0; } -rect.unknown, path.unknown { +rect.absent, path.absent { fill: #727272; stroke-width: 0; } +rect.extra, path.extra { + fill: #cc0000; + stroke-width: 0; +} + g.tooltip text { font-size: 13px; } diff --git a/database/makeHTML.py b/database/makeHTML.py index a0606ef..d2750eb 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -64,21 +64,26 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): """.format(request_set[2], requests[i][0])) statuses = ["new", "validating", "validated", "defined", "approved", - "submitted", "done", "unkown"] + "submitted", "done", "absent", "extra"] + sum_known = requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4] + requests[i][5] + requests[i][6] + requests[i][7] + total = max(request_set[4], sum_known) x_pos = 0.0 tmp_object_counter = object_counter for j in range(7): bar_width = 0.0 - if request_set[4] != 0: - bar_width = float(requests[i][j+1])/float(request_set[4])*100.0 + if total != 0: + bar_width = float(requests[i][j+1])/float(total)*100.0 fout.write("""\ """.format(x_pos, bar_width, statuses[j], tmp_object_counter)) x_pos += bar_width tmp_object_counter += 1 + unkown_reason = "absent" + if total > request_set[4]: + unkown_reason = "extra" fout.write("""\ - -""".format(x_pos, 100.0 - x_pos, tmp_object_counter)) + +""".format(x_pos, 100.0 - x_pos, unkown_reason, tmp_object_counter)) fout.write("""\ @@ -86,7 +91,6 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): for j in range(7): fout.write("""\ - {2} {1} @@ -94,26 +98,72 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): object_counter += 1 fout.write("""\ - - - {1} unkown + + {2} {1} -""".format(object_counter, 0)) +""".format(object_counter, unkown_reason, total - sum_known)) object_counter += 1 fout.write("""\ """) fout.write(" \n") elif page == 1: - # fout.write(" {3}/{4}\n".format( - # campaign_classes[i + class_offset], request_set[2], - # requests[i][0], requests[i][7], request_set[4])) fout.write(" \n".format(campaign_classes[i + class_offset])) - fout.write(" \"{2}\n".format( - request_set[2], requests[i][0], - requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4], - requests[i][5], requests[i][6], requests[i][7], - request_set[4])) + fout.write("""\ + + + +""".format(request_set[2], requests[i][0])) + statuses = ["new", "validating", "validated", "defined", "approved", + "submitted", "done", "absent", "extra"] + sum_known = requests[i][1] + requests[i][2] + requests[i][3] + requests[i][4] + requests[i][5] + requests[i][6] + requests[i][7] + total = max(request_set[4], sum_known) + combined_statuses = ["preparation", "ready to run", "running", + "done", "absent", "extra"] + combined_classes = [statuses[0], statuses[4], statuses[5], statuses[6]] + combined_requests = [requests[i][1] + requests[i][2] + requests[i][3], + requests[i][4] + requests[i][5], requests[i][6], + requests[i][7]] + x_pos = 0.0 + tmp_object_counter = object_counter + for j in range(4): + bar_width = 0.0 + if total != 0: + bar_width = float(combined_requests[j])/float(total)*100.0 + fout.write("""\ + +""".format(x_pos, bar_width, combined_classes[j], tmp_object_counter)) + x_pos += bar_width + tmp_object_counter += 1 + unkown_reason = "absent" + if total > request_set[4]: + unkown_reason = "extra" + fout.write("""\ + +""".format(x_pos, 100.0 - x_pos, unkown_reason, tmp_object_counter)) + fout.write("""\ + + +""") + for j in range(4): + fout.write("""\ + + + {2} {3} + +""".format(object_counter, combined_classes[j], combined_requests[j], + combined_statuses[j])) + object_counter += 1 + fout.write("""\ + + + {2} {1} + +""".format(object_counter, unkown_reason, total - sum_known)) + object_counter += 1 + fout.write("""\ + +""") fout.write(" \n") return @@ -257,6 +307,24 @@ def makeAnalyzerHTML(): + + + + From 8bf8a52002a06161411209865a1649c2d834e5b5 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Wed, 29 Jun 2016 23:13:37 +0200 Subject: [PATCH 62/64] Hide notes. Center sprites. --- database/global.css | 44 +++++++++++++++++++++++---- database/makeHTML.py | 70 ++++++++++++++++++++++++++++++++----------- database/sprites.png | Bin 936 -> 1893 bytes 3 files changed, 90 insertions(+), 24 deletions(-) diff --git a/database/global.css b/database/global.css index 3d7cdab..26666e9 100644 --- a/database/global.css +++ b/database/global.css @@ -6,7 +6,7 @@ *************************/ body { - background-color: #ffffff; + background-color: #fbfbfb; padding: 0; margin: 0; border: 0; @@ -57,7 +57,7 @@ th.lhe, th.gs, th.dr, th.miniaod, th.miniaodv2 { } td.lhe, td.gs, td.dr, td.miniaod, td.miniaodv2 { - /*text-align: center;*/ + text-align: center; } p.update-time { @@ -128,10 +128,42 @@ g.tooltip text { font-size: 13px; } -div.spreadsheet_icon { - width: 32px; - height: 32px; - background: url(sprites.png) -5px -5px; +span.spreadsheet_icon { + display: block; + width: 30px; + height: 30px; + background: url(sprites.png) -6px -6px; + margin: auto; +} + +span.spreadsheet_icon:hover { + background: url(sprites.png) -48px -6px; +} + +span.note_icon { + display:block; + width: 30px; + height: 30px; + background: url(sprites.png) -6px -48px; + margin: auto; +} + +span.note_icon:hover { + background: url(sprites.png) -48px -48px; +} + +span.note_content { + display: none; + background-color: #f2f2ee; + border: 1px solid; + border-color: #000000; + position: absolute; + padding: 7px; + -moz-transform: translateX(-100%) translateY(-15px); + -ms-transform: translateX(-100%) translateY(-15px); + -webkit-transform: translateX(-100%) translateY(-15px); + -o-transform: translateX(-100%) translateY(-15px); + transform: translateX(-100%) translateY(-15px); } td.spreadsheet a:hover { diff --git a/database/makeHTML.py b/database/makeHTML.py index d2750eb..4987436 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -73,6 +73,8 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): bar_width = 0.0 if total != 0: bar_width = float(requests[i][j+1])/float(total)*100.0 + if bar_width < 0.0: + bar_width = 0.0 fout.write("""\ """.format(x_pos, bar_width, statuses[j], tmp_object_counter)) @@ -83,7 +85,7 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): unkown_reason = "extra" fout.write("""\ -""".format(x_pos, 100.0 - x_pos, unkown_reason, tmp_object_counter)) +""".format(x_pos, max(100.0 - x_pos, 0.0), unkown_reason, tmp_object_counter)) fout.write("""\ @@ -91,15 +93,15 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): for j in range(7): fout.write("""\ - - {2} {1} + + {2} {1} """.format(object_counter, statuses[j], requests[i][j+1])) object_counter += 1 fout.write("""\ - - {2} {1} + + {2} {1} """.format(object_counter, unkown_reason, total - sum_known)) object_counter += 1 @@ -130,6 +132,8 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): bar_width = 0.0 if total != 0: bar_width = float(combined_requests[j])/float(total)*100.0 + if bar_width < 0.0: + bar_width = 0.0 fout.write("""\ """.format(x_pos, bar_width, combined_classes[j], tmp_object_counter)) @@ -140,7 +144,7 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): unkown_reason = "extra" fout.write("""\ -""".format(x_pos, 100.0 - x_pos, unkown_reason, tmp_object_counter)) +""".format(x_pos, max(100.0 - x_pos, 0.0), unkown_reason, tmp_object_counter)) fout.write("""\ @@ -148,16 +152,16 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): for j in range(4): fout.write("""\ - - {2} {3} + + {2} {3} """.format(object_counter, combined_classes[j], combined_requests[j], combined_statuses[j])) object_counter += 1 fout.write("""\ - - {2} {1} + + {2} {1} """.format(object_counter, unkown_reason, total - sum_known)) object_counter += 1 @@ -203,22 +207,22 @@ def writeInstances(page, fout, c, super_campaign, request_set): if request_set[6] == "": fout.write("  \n") else: - fout.write("
 
\n".format( + fout.write("  
\n".format( request_set[6])) if request_set[5] == "": fout.write("  \n") else: - fout.write(" {0}\n".format(request_set[5])) + fout.write("  {0}\n".format(request_set[5])) elif page == 1: if request_set[6] == "": fout.write("  \n") else: - fout.write("
 
\n".format( + fout.write("  
\n".format( request_set[6])) if request_set[5] == "": fout.write("  \n") else: - fout.write(" {0}\n".format(request_set[5])) + fout.write("  {0}\n".format(request_set[5])) fout.write("\n") return @@ -325,6 +329,18 @@ def makeAnalyzerHTML(): ]]> + + + @@ -332,12 +348,12 @@ def makeAnalyzerHTML():

Exotica MC

- + """) @@ -400,6 +416,22 @@ def makeContactHTML(): ]]> + + + @@ -407,7 +439,7 @@ def makeContactHTML():

Exotica MC

-
+ + +

Go to analyzer page

""") diff --git a/database/sprites.png b/database/sprites.png index afc1305691b422e45bb8eee07374608b592f5740..5470b64123eeb381e293d3204deb5b7374f18943 100644 GIT binary patch delta 1891 zcmV-p2b}n*2jvcs7k^X;1^@s69BUB400006VoOIv0QLa?00f7^?sfnG010qNS#tmY z3ljhU3ljkVnw%H_000McNliru;07HPFAHdZOTGXA2J%TnK~#9!?OR_+Q*9i6&N;I& zohig-UQ|e!e6)hZAm~ct3(Jcoqx?ff%piy^T2XgB`eCU0g9RFfcGMFfcGMu);nGaYp>=+1lEQI`awK zvRzM0pbZTTG0&Z&Dr9qkj1go;b(`wUCsxHSo};SOL8y`)kq&i6`%FHOH^-2Kj=d;~ z{Mxl^D}Qx5-2uH`zdI)ZjR&h z0C-C2ILC2r08TgOS zYD$V!q*9ejl5~yZxRRekGn7&<0Q@9Y3jmbTcYi$3pAG~9PZFWfxO>dy<>d$xX)qX2 zT3QOVS`7g3`~B$d?nZBKFV?SLzpbF4;L+Q+Z?As&@}<#eG%8Mnf*`~lC#4h|$Dz2m z80XKQM|E|zR-@5q0l??;p|i6SM~@yI;5cp@rSy>?2v-RqjQ|2GQK+J(J;!mVt*wPd zqkoC`zR%}FM@I)F>BmkQ92|Vm+uM84Xf%FTqEID-0>GX83FP9}Vlz-H+vNC-7^eH%Wn-EeLEw8SqYJZQKO)1&(k|g2Ckt1@>6n{k# zLguTTbpbw34ACd z-ZjBAvK;pB-;c+SABUOdVj(QX$jHdQ01BgAmLxof?%lgLj8Kl_;P?CG*98RyOV)&r z`Fy^xo134X_x1PppGveMuUxr;tgI~g_0Z4|=H}+)*T;?>Te9Zt*|Rd4Gk-HP7#bQ1 z^Hf5}skjvy%SpRn{tN!y)6;`pyLJHps;a8c)zt;B*9!p9>-DIrs*>lwdGjW0eRp^F zxqbWgeO6GU$2H&Y_v8Nk`>3g@kqs*Je$8fcm=WE(cP})5f>L@8z~{J3X^yIJ%XU5P z`i>BS`uck0v}qGwzkZFp zyu7IMCMG7ZW5*7B{rXk*YeLBJXl=t4*P-+C^XTa4ke?#Y^Qft*iCWmVx3|l=YhYmD zuE*o4F&d5YN(%c}>renNnM@Ex5dct8QGw3R&Zvcbb#*nKKYtz^x_?UusR1yb)PCri zHEU2`UyuC!e7StG+wE|>-C?UU-F0M_3#>H`5KoA5K z01OWg_w@DkHC0qpxPMmK98vn!b8*uqXqcm;qjFP+Qrg4wd{ZD0aHXw%x7f6Jo`)n! z`1bAF;Mmxh+w1i@r>3SJilRs}<%l^U1jEC_P^;D0ym>SF`}=JGoRTCxlq4yx2d20V zjTbhX?GL?PkF8s`CiL3BszVhtceoXa2n>nswXUQN&4?RNWn9+J1eiiorO-_KRa^>l zC{rlQp)7|oB%-kAR>KVtLO7<--x$jhnvR28N~VE4#agpjUBj6e5?!% d3=9mh;XnN1{Ngu}Dro=!002ovPDHLkV1i7LnWF#z delta 927 zcmV;Q17Q5+4yXr^7k?@U1^@s6Ekk^8000ATNklXMo+$mW};px1{ir#uaJ1SYb~AHm*aNt zJs)@DU3cHnIeV|Y_CCM!Uu&Ix&LzZOPy!UxTXcp9G_VLF(0>RtFoD4CnkuxvI&j#4 zLxJbY<&J4M2;`*pmvaR4OY~^(+xB@~}Rc*}LS}gPoBao7t#yZg;I#yO$Sg zA)xQKUCiq@mw#RVxxuD(MT!zLmoA`Y_MC{0iO5v|U;JXGGxJ6>J4i$e60QKwh{#R= zwX6&4WuW&(WCMUtEg_qqYEJ-V=6`JfswwybB67)}PjSv21yIjF z7ghBjfbU5^Gj9QKp+45k_Nwa1RyVTiftJXj6CyGTz~=;tbM6EY?MnkPvlFVi-?zVf|%l=R7 z<$rS%>Cmv0LMPK>Q-{|de0bLb4U|KFf)$aDJ`hHr4I2F?Ktshzh$HQP|8hGlC-F3Y}J0=j=K?M3YfX2Tw zmZ#j&uAuz1|L@2MH1NHRKqJt=1OmH@3K$Ix^auO1FG)0n#X0~0002ovPDHLkV1l8E BwPOGP From e0dfd848856a8681af317afebcd55f737f47abb7 Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Thu, 30 Jun 2016 01:59:53 +0200 Subject: [PATCH 63/64] Updated CSS style. Add table of contents. --- database/global.css | 112 +++++++++++++++++++++++++++++++++---------- database/makeHTML.py | 45 +++++++++-------- 2 files changed, 108 insertions(+), 49 deletions(-) diff --git a/database/global.css b/database/global.css index 26666e9..47574f0 100644 --- a/database/global.css +++ b/database/global.css @@ -5,6 +5,12 @@ ** *************************/ +/**************** +** +** Page +** +****************/ + body { background-color: #fbfbfb; padding: 0; @@ -18,62 +24,114 @@ div.wrapper { margin: 20px 50px 20px 50px; } -a { +/**************** +** +** Text +** +****************/ + +a, a:visited { text-decoration: none; + color: #0000dd; +} + +a:hover, a:visited:hover { + color: #dd0000; +} + +h1 { + font-size: 32pt; + color: #000000; } -a:hover { - text-decoration: underline; +h2 { + font-size: 24pt; + background-color: #84a4aa; + color: #000000; + padding: 6pt; + margin-top: 24pt; +} + +div.toc h2 { + font-size: 18pt; + background-color: #fbfbfb; + color: #000000; + padding: 0px; + margin-bottom: 10pt; + margin-top: 0px; +} + +li { + margin: 10pt 0pt 10pt 0pt; +} + +div.update-time { + color: #999999; + font-size: 10pt; + font-style: italic; + margin: 15pt 0 15pt 0; } +/**************** +** +** Table +** +****************/ + table { border-collapse: collapse; } -tr:nth-child(even) { +tbody tr:nth-child(odd) { background: #ffffff; } -tr:nth-child(odd) { +tbody tr:nth-child(even) { background: #dddddd; } -tr.table_header { - background-color: #999999; +thead { border-bottom: 2px solid black; - text-color: #ffffff; +} + +thead tr { + background-color: #c0bcb9; + color: #000000; } th { - padding: 2px; + padding: 8pt 2pt 6pt 2pt; } td { - padding: 8px 2px 8px 2px; + padding-top: 8px; + padding-bottom: 8px; } th.lhe, th.gs, th.dr, th.miniaod, th.miniaodv2 { min-width: 100px; } -td.lhe, td.gs, td.dr, td.miniaod, td.miniaodv2 { - text-align: center; +th.notes { + padding-right: 5pt; } -p.update-time { - color: #999999; - font-size: 0.8em; - font-style: italic; +td.lhe, td.gs, td.dr, td.miniaod, td.miniaodv2 { + text-align: center; + padding-left: 2px; + padding-right: 2px; } -img { - border: 0px; +td.process, td.tag, td.requester, td.contact, td.events { + padding-left: 5pt; + padding-right: 5pt; } -img.spreadsheet_icon { - width: 21px; - height: 21px; -} +/**************** +** +** Bar charts +** +****************/ svg.status { margin-top: -18px; @@ -128,6 +186,12 @@ g.tooltip text { font-size: 13px; } +/**************** +** +** Sprites +** +****************/ + span.spreadsheet_icon { display: block; width: 30px; @@ -165,7 +229,3 @@ span.note_content { -o-transform: translateX(-100%) translateY(-15px); transform: translateX(-100%) translateY(-15px); } - -td.spreadsheet a:hover { - text-decoration: none; -} \ No newline at end of file diff --git a/database/makeHTML.py b/database/makeHTML.py index 4987436..5edc9a7 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -255,12 +255,28 @@ def writeSuperCampaigns(page, fout, c): FROM SuperCampaigns ORDER BY Active;""") super_campaigns = c.fetchall() + fout.write("""\ +
+

Campaigns:

+
    +""") + for super_campaign in super_campaigns: + fout.write("""\ +
  • {1}
    • +""".format(super_campaign[1].replace(" ", "-"), super_campaign[1])) + fout.write("""\ +
+
+ +""") + for super_campaign in super_campaigns: fout.write("""\ -

{0}

+

{1}

+ -""".format(super_campaign[1])) +""".format(super_campaign[1].replace(" ", "-"), super_campaign[1])) if page == 0: fout.write("""\ @@ -286,11 +302,11 @@ def writeSuperCampaigns(page, fout, c): """) - fout.write("\n") + fout.write("\n\n\n") writeRequestSets(page, fout, c, super_campaign) - fout.write("
ProcessSpreadsheet Notes
\n") + fout.write("\n\n") return @@ -348,13 +364,6 @@ def makeAnalyzerHTML():

Exotica MC

- - """) conn = sqlite3.connect(mcmscripts_config.database_location) @@ -369,7 +378,7 @@ def makeAnalyzerHTML(): check_time = c.fetchone() fout.write("""\ -

Updated {0}

+
Updated {0}

@@ -439,16 +448,6 @@ def makeContactHTML():

Exotica MC

- -

Go to analyzer page

""") @@ -470,7 +469,7 @@ def makeContactHTML(): check_time = c.fetchone() fout.write("""\ -

Updated {0}

+
Updated {0}

From 522a3cc3f98bb6a8d4720528f80d1862d370861f Mon Sep 17 00:00:00 2001 From: David Sheffield Date: Tue, 5 Jul 2016 18:24:16 +0200 Subject: [PATCH 64/64] Add number of requests to pages. Add extra requests to bar display. --- database/makeHTML.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/database/makeHTML.py b/database/makeHTML.py index 5edc9a7..5d0552a 100755 --- a/database/makeHTML.py +++ b/database/makeHTML.py @@ -72,7 +72,10 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): for j in range(7): bar_width = 0.0 if total != 0: - bar_width = float(requests[i][j+1])/float(total)*100.0 + if requests[i][j+1] <= request_set[4]: + bar_width = float(requests[i][j+1])/float(total)*100.0 + else: + bar_width = float(request_set[4])/float(total)*100.0 if bar_width < 0.0: bar_width = 0.0 fout.write("""\ @@ -83,6 +86,8 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): unkown_reason = "absent" if total > request_set[4]: unkown_reason = "extra" + # if x_pos >= 95.0: + # x_pos = 95.0 fout.write("""\ """.format(x_pos, max(100.0 - x_pos, 0.0), unkown_reason, tmp_object_counter)) @@ -103,7 +108,7 @@ def writeRequests(page, fout, c, super_campaign, request_set, instance): {2} {1} -""".format(object_counter, unkown_reason, total - sum_known)) +""".format(object_counter, unkown_reason, max(total - sum_known, sum_known - request_set[4]))) object_counter += 1 fout.write("""\ @@ -195,8 +200,9 @@ def writeInstances(page, fout, c, super_campaign, request_set): {3} {4} {5} + {6} """.format(request_set[1], request_set[2], instance[3], instance[2], - instance[1], display_number(request_set[3]))) + instance[1], display_number(request_set[3]), request_set[4])) elif page == 1: fout.write("""\ {0} @@ -284,6 +290,7 @@ def writeSuperCampaigns(page, fout, c): Requester Contact Events + Requests LHE GS DR