~maddevelopers/mg5amcnlo/dim6_eft

« back to all changes in this revision

Viewing changes to madgraph/interface/amcatnlo_run_interface.py

  • Committer: olivier Mattelaer
  • Date: 2014-09-24 20:33:52 UTC
  • mfrom: (253.17.56 2.2.0)
  • Revision ID: olivier.mattelaer@uclouvain.be-20140924203352-bplvftjd8czlhagt
pass to 2.2.0

Show diffs side-by-side

added added

removed removed

Lines of Context:
35
35
import tarfile
36
36
import copy
37
37
import datetime
 
38
import tarfile
38
39
 
39
40
try:
40
41
    import readline
403
404
        """Check the argument for the plot command
404
405
        plot run_name modes"""
405
406
 
406
 
        if options['force']:
407
 
            self.force = True
408
407
 
409
408
        madir = self.options['madanalysis_path']
410
409
        td = self.options['td_path']
965
964
        self.check_plot(args)
966
965
        logger.info('plot for run %s' % self.run_name)
967
966
        
968
 
        self.ask_edit_cards([], args, plot=True)
 
967
        if not self.force:
 
968
            self.ask_edit_cards([], args, plot=True)
969
969
                
970
970
        if any([arg in ['parton'] for arg in args]):
971
971
            filename = pjoin(self.me_dir, 'Events', self.run_name, 'events.lhe')
972
972
            if os.path.exists(filename+'.gz'):
973
 
                os.system('gunzip -f %s' % (filename+'.gz') )
 
973
                misc.gunzip(filename)
974
974
            if  os.path.exists(filename):
975
975
                logger.info('Found events.lhe file for run %s' % self.run_name) 
976
976
                shutil.move(filename, pjoin(self.me_dir, 'Events', 'unweighted_events.lhe'))
977
977
                self.create_plot('parton')
978
978
                shutil.move(pjoin(self.me_dir, 'Events', 'unweighted_events.lhe'), filename)
979
 
                os.system('gzip -f %s' % filename)
 
979
                misc.gzip(filename)
980
980
                
981
981
        if any([arg in ['all','parton'] for arg in args]):
982
982
            filename = pjoin(self.me_dir, 'Events', self.run_name, 'MADatNLO.top')
1020
1020
                                self.run_name)
1021
1021
                    return
1022
1022
                filename = filenames[0]
1023
 
                os.system('gunzip -c -f %s > %s' % (filename,
1024
 
                          pjoin(self.me_dir, 'Events','pythia_events.hep')))
1025
 
 
 
1023
                misc.gunzip(filename, keep=True, stdout=pjoin(self.me_dir, 'Events','pythia_events.hep'))
 
1024
                
1026
1025
                if not os.path.exists(pjoin(self.me_dir, 'Cards', 'pythia_card.dat')):
1027
 
                    files.cp(pjoin(self.me_dir, 'Cards', 'pythia_card_default.dat'),
 
1026
                    if aMCatNLO and not self.options['mg5_path']:
 
1027
                        raise "plotting NLO HEP file needs MG5 utilities"
 
1028
                    
 
1029
                    files.cp(pjoin(self.options['mg5_path'], 'Template','LO', 'Cards', 'pythia_card_default.dat'),
1028
1030
                             pjoin(self.me_dir, 'Cards', 'pythia_card.dat'))
1029
1031
                self.run_hep2lhe()
1030
1032
            else:
1031
1033
                filename = filenames[0]
1032
 
                os.system('gunzip -c -f %s > %s' % (filename,
1033
 
                          pjoin(self.me_dir, 'Events','pythia_events.lhe')))
1034
 
            self.create_plot('Pythia')
 
1034
                misc.gunzip(filename, keep=True, stdout=pjoin(self.me_dir, 'Events','pythia_events.hep'))
 
1035
 
 
1036
            self.create_plot('shower')
1035
1037
            lhe_file_name = filename.replace('.hep.gz', '.lhe')
1036
1038
            shutil.move(pjoin(self.me_dir, 'Events','pythia_events.lhe'), 
1037
1039
                        lhe_file_name)
1038
 
            os.system('gzip -f %s' % lhe_file_name)
 
1040
            misc.gzip(lhe_file_name)
1039
1041
                    
1040
1042
        if any([arg in ['all','pgs'] for arg in args]):
1041
1043
            filename = pjoin(self.me_dir, 'Events', self.run_name, 
1042
1044
                                            '%s_pgs_events.lhco' % self.run_tag)
1043
1045
            if os.path.exists(filename+'.gz'):
1044
 
                os.system('gunzip -f %s' % (filename+'.gz') )
 
1046
                misc.gunzip(filename)
1045
1047
            if  os.path.exists(filename):
1046
1048
                self.create_plot('PGS')
1047
 
                os.system('gzip -f %s' % filename)                
 
1049
                misc.gzip(filename)                
1048
1050
            else:
1049
1051
                logger.info('No valid files for pgs plot')
1050
1052
                
1052
1054
            filename = pjoin(self.me_dir, 'Events', self.run_name, 
1053
1055
                                        '%s_delphes_events.lhco' % self.run_tag)
1054
1056
            if os.path.exists(filename+'.gz'):
1055
 
                os.system('gunzip -f %s' % (filename+'.gz') )
 
1057
                misc.gunzip(filename)
1056
1058
            if  os.path.exists(filename):
1057
1059
                #shutil.move(filename, pjoin(self.me_dir, 'Events','delphes_events.lhco'))
1058
1060
                self.create_plot('Delphes')
1059
1061
                #shutil.move(pjoin(self.me_dir, 'Events','delphes_events.lhco'), filename)
1060
 
                os.system('gzip -f %s' % filename)                
 
1062
                misc.gzip(filename)                
1061
1063
            else:
1062
1064
                logger.info('No valid files for delphes plot')
1063
1065
 
1121
1123
        this function just wraps the do_launch one"""
1122
1124
        self.do_launch(line)
1123
1125
 
 
1126
 
1124
1127
    ############################################################################
1125
1128
    def do_treatcards(self, line, amcatnlo=True):
1126
1129
        """Advanced commands: this is for creating the correct run_card.inc from the nlo format"""
 
1130
                #check if no 'Auto' are present in the file
 
1131
        self.check_param_card(pjoin(self.me_dir, 'Cards','param_card.dat'))
1127
1132
        return super(aMCatNLOCmd,self).do_treatcards(line, amcatnlo)
1128
1133
    
1129
1134
    ############################################################################
1229
1234
 
1230
1235
        self.update_status('', level='all', update_results=True)
1231
1236
 
 
1237
    def print_results_in_shell(self, data):
 
1238
        """Have a nice results prints in the shell,
 
1239
        data should be of type: gen_crossxhtml.OneTagResults"""
 
1240
        if not data:
 
1241
            return
 
1242
        logger.info("  === Results Summary for run: %s tag: %s ===\n" % (data['run_name'],data['tag']))
 
1243
        if self.ninitial == 1:
 
1244
            logger.info("     Width :   %.4g +- %.4g GeV" % (data['cross'], data['error']))
 
1245
        else:
 
1246
            logger.info("     Cross-section :   %.4g +- %.4g pb" % (data['cross'], data['error']))
 
1247
        logger.info("     Nb of events :  %s" % data['nb_event'] )
 
1248
        #if data['cross_pythia'] and data['nb_event_pythia']:
 
1249
        #    if self.ninitial == 1:
 
1250
        #        logger.info("     Matched Width :   %.4g +- %.4g GeV" % (data['cross_pythia'], data['error_pythia']))
 
1251
        #    else:
 
1252
        #        logger.info("     Matched Cross-section :   %.4g +- %.4g pb" % (data['cross_pythia'], data['error_pythia']))            
 
1253
        #    logger.info("     Nb of events after Matching :  %s" % data['nb_event_pythia'])
 
1254
        #    if self.run_card['use_syst'] in self.true:
 
1255
        #        logger.info("     Be carefull that matched information are here NOT for the central value. Refer to SysCalc output for it")    
 
1256
        logger.info(" " )
 
1257
 
 
1258
    def print_results_in_file(self, data, path, mode='w'):
 
1259
        """Have a nice results prints in the shell,
 
1260
        data should be of type: gen_crossxhtml.OneTagResults"""
 
1261
        if not data:
 
1262
            return
 
1263
        
 
1264
        fsock = open(path, mode)
 
1265
        
 
1266
        fsock.write("  === Results Summary for run: %s tag: %s  process: %s ===\n" % \
 
1267
                    (data['run_name'],data['tag'], os.path.basename(self.me_dir)))
 
1268
        
 
1269
        if self.ninitial == 1:
 
1270
            fsock.write("     Width :   %.4g +- %.4g GeV\n" % (data['cross'], data['error']))
 
1271
        else:
 
1272
            fsock.write("     Cross-section :   %.4g +- %.4g pb\n" % (data['cross'], data['error']))
 
1273
        fsock.write("     Nb of events :  %s\n" % data['nb_event'] )
 
1274
        #if data['cross_pythia'] and data['nb_event_pythia']:
 
1275
        #    if self.ninitial == 1:
 
1276
        #        fsock.write("     Matched Width :   %.4g +- %.4g GeV\n" % (data['cross_pythia'], data['error_pythia']))
 
1277
        #    else:
 
1278
        #        fsock.write("     Matched Cross-section :   %.4g +- %.4g pb\n" % (data['cross_pythia'], data['error_pythia']))            
 
1279
        #    fsock.write("     Nb of events after Matching :  %s\n" % data['nb_event_pythia'])
 
1280
        fsock.write(" \n" )
 
1281
 
 
1282
 
 
1283
 
1232
1284
 
1233
1285
 
1234
1286
    def update_random_seed(self):
1245
1297
 
1246
1298
 
1247
1299
    def get_characteristics(self, file):
1248
 
        """reads the proc_characteristics file and initialises the correspondent
 
1300
        """reads the proc_characteristics file and initialises the correspondant
1249
1301
        dictionary"""
1250
1302
        lines = [l for l in open(file).read().split('\n') if l and not l.startswith('#')]
1251
1303
        self.proc_characteristics = {}
1261
1313
        if not 'only_generation' in options.keys():
1262
1314
            options['only_generation'] = False
1263
1315
 
 
1316
        if mode in ['LO', 'NLO'] and self.run_card['iappl'] == '2' and not options['only_generation']:
 
1317
            options['only_generation'] = True
1264
1318
        self.get_characteristics(pjoin(self.me_dir, 'SubProcesses', 'proc_characteristics'))
1265
1319
 
1266
1320
        if self.cluster_mode == 1:
1317
1371
 
1318
1372
                if not options['only_generation'] and not options['reweightonly']:
1319
1373
                    to_always_rm.extend(to_rm)
 
1374
                    if os.path.exists(pjoin(self.me_dir, 'SubProcesses', dir,'MadLoop5_resources.tar.gz')):
 
1375
                        to_always_rm.append(pjoin(self.me_dir, 'SubProcesses', dir,'MadLoop5_resources.tar.gz'))
1320
1376
                files.rm([pjoin(self.me_dir, 'SubProcesses', dir, d) for d in to_always_rm])
1321
1377
 
1322
1378
        mcatnlo_status = ['Setting up grid', 'Computing upper envelope', 'Generating events']
1323
1379
 
 
1380
        if self.run_card['iappl']=='2':
 
1381
            self.applgrid_distribute(options,mode,p_dirs)
 
1382
 
1324
1383
        if options['reweightonly']:
1325
1384
            event_norm=self.run_card['event_norm']
1326
1385
            nevents=int(self.run_card['nevents'])
1327
 
            self.reweight_and_collect_events(options, mode, nevents, event_norm)
1328
 
            return
 
1386
            return self.reweight_and_collect_events(options, mode, nevents, event_norm)
1329
1387
 
1330
1388
        devnull = os.open(os.devnull, os.O_RDWR) 
1331
1389
        if mode in ['LO', 'NLO']:
1410
1468
                
1411
1469
            cross, error = sum_html.make_all_html_results(self, folder_names[mode])
1412
1470
            self.results.add_detail('cross', cross)
1413
 
            self.results.add_detail('error', error) 
 
1471
            self.results.add_detail('error', error)
 
1472
            if self.run_card['iappl'] != '0':
 
1473
                self.applgrid_combine(cross,error)
1414
1474
            self.update_status('Run complete', level='parton', update_results=True)
1415
1475
 
1416
1476
            return
1529
1589
        return self.reweight_and_collect_events(options, mode, nevents, event_norm)
1530
1590
 
1531
1591
 
 
1592
    def applgrid_combine(self,cross,error):
 
1593
        """Combines the APPLgrids in all the SubProcess/P*/all_G*/ directories"""
 
1594
        logger.debug('Combining APPLgrids \n')
 
1595
        applcomb=pjoin(self.options['applgrid'].rstrip('applgrid-config'),'applgrid-combine')
 
1596
        with open(pjoin(self.me_dir,'SubProcesses','dirs.txt')) as dirf:
 
1597
            all_jobs=dirf.readlines()
 
1598
        ngrids=len(all_jobs)
 
1599
        nobs  =len([name for name in os.listdir(pjoin(self.me_dir,'SubProcesses',all_jobs[0].rstrip())) \
 
1600
                        if name.endswith("_out.root")])
 
1601
        for obs in range(0,nobs):
 
1602
            gdir = [pjoin(self.me_dir,'SubProcesses',job.rstrip(),"grid_obs_"+str(obs)+"_out.root") for job in all_jobs]
 
1603
            # combine APPLgrids from different channels for observable 'obs'
 
1604
            if self.run_card["iappl"] == "1":
 
1605
                misc.call([applcomb,'-o', pjoin(self.me_dir,"Events",self.run_name,"aMCfast_obs_"+str(obs)+"_starting_grid.root"), '--optimise']+ gdir)
 
1606
            elif self.run_card["iappl"] == "2":
 
1607
                unc2_inv=pow(cross/error,2)
 
1608
                unc2_inv_ngrids=pow(cross/error,2)*ngrids
 
1609
                misc.call([applcomb,'-o', pjoin(self.me_dir,"Events",self.run_name,"aMCfast_obs_"+str(obs)+".root"),'-s',str(unc2_inv),'--weight',str(unc2_inv)]+ gdir)
 
1610
                for job in all_jobs:
 
1611
                    os.remove(pjoin(self.me_dir,'SubProcesses',job.rstrip(),"grid_obs_"+str(obs)+"_in.root"))
 
1612
            else:
 
1613
                raise aMCatNLOError('iappl parameter can only be 0, 1 or 2')
 
1614
            # after combining, delete the original grids
 
1615
            for ggdir in gdir:
 
1616
                os.remove(ggdir)
 
1617
 
 
1618
        
 
1619
    def applgrid_distribute(self,options,mode,p_dirs):
 
1620
        """Distributes the APPLgrids ready to be filled by a second run of the code"""
 
1621
        # if no appl_start_grid argument given, guess it from the time stamps of the starting grid files
 
1622
        if not('appl_start_grid' in options.keys() and options['appl_start_grid']):
 
1623
            gfiles=glob.glob(pjoin(self.me_dir, 'Events','*','aMCfast_obs_0_starting_grid.root'))
 
1624
            time_stamps={}
 
1625
            for root_file in gfiles:
 
1626
                time_stamps[root_file]=os.path.getmtime(root_file)
 
1627
            options['appl_start_grid']= \
 
1628
                max(time_stamps.iterkeys(), key=(lambda key: time_stamps[key])).split('/')[-2]
 
1629
            logger.info('No --appl_start_grid option given. Guessing that start grid from run "%s" should be used.' \
 
1630
                            % options['appl_start_grid'])
 
1631
 
 
1632
        if 'appl_start_grid' in options.keys() and options['appl_start_grid']:
 
1633
            self.appl_start_grid = options['appl_start_grid']
 
1634
            start_grid_dir=pjoin(self.me_dir, 'Events', self.appl_start_grid)
 
1635
            # check that this dir exists and at least one grid file is there
 
1636
            if not os.path.exists(pjoin(start_grid_dir,'aMCfast_obs_0_starting_grid.root')):
 
1637
                raise self.InvalidCmd('APPLgrid file not found: %s' % \
 
1638
                                  pjoin(start_grid_dir,'aMCfast_obs_0_starting_grid.root'))
 
1639
            else:
 
1640
                all_grids=[pjoin(start_grid_dir,name) for name in os.listdir(start_grid_dir) \
 
1641
                               if name.endswith("_starting_grid.root")]
 
1642
                nobs =len(all_grids)
 
1643
                gstring=" ".join(all_grids)
 
1644
        if not hasattr(self, 'appl_start_grid') or not self.appl_start_grid:
 
1645
            raise self.InvalidCmd('No APPLgrid name currently defined. Please provide this information.')             
 
1646
        if mode == 'NLO':
 
1647
            gdir='all_G'
 
1648
        elif mode == 'LO':
 
1649
            gdir='born_G'
 
1650
        #copy the grid to all relevant directories
 
1651
        for pdir in p_dirs:
 
1652
            g_dirs = [file for file in os.listdir(pjoin(self.me_dir,"SubProcesses",pdir)) \
 
1653
                      if file.startswith(gdir) and os.path.isdir(pjoin(self.me_dir,"SubProcesses",pdir, file))]
 
1654
            for g_dir in g_dirs:
 
1655
                for grid in all_grids:
 
1656
                    obs=grid.split('_')[-3]
 
1657
                    files.cp(grid,pjoin(self.me_dir,"SubProcesses",pdir,g_dir,'grid_obs_'+obs+'_in.root'))
 
1658
 
 
1659
 
1532
1660
    def collect_log_files(self, folders, istep):
1533
1661
        """collect the log files and put them in a single, html-friendly file inside the run_...
1534
1662
        directory"""
1617
1745
        proc_card_lines = open(pjoin(self.me_dir, 'Cards', 'proc_card_mg5.dat')).read().split('\n')
1618
1746
        process = ''
1619
1747
        for line in proc_card_lines:
1620
 
            if line.startswith('generate'):
1621
 
                process = line.replace('generate ', '')
 
1748
            if line.startswith('generate') or line.startswith('add process'):
 
1749
                process = process+(line.replace('generate ', '')).replace('add process ','')+' ; '
1622
1750
        lpp = {'0':'l', '1':'p', '-1':'pbar'}
1623
1751
        proc_info = '\n      Process %s\n      Run at %s-%s collider (%s + %s GeV)' % \
1624
 
        (process, lpp[self.run_card['lpp1']], lpp[self.run_card['lpp2']], 
 
1752
        (process[:-3], lpp[self.run_card['lpp1']], lpp[self.run_card['lpp2']], 
1625
1753
                self.run_card['ebeam1'], self.run_card['ebeam2'])
1626
1754
        
1627
1755
        # Gather some basic statistics for the run and extracted from the log files.
1659
1787
                          '\n      Total cross-section: %(xsect)8.3e +- %(errt)6.1e pb' % \
1660
1788
                        self.cross_sect_dict
1661
1789
 
1662
 
                if int(self.run_card['nevents'])>=10000 and self.run_card['reweight_scale']=='.true.':
 
1790
                if int(self.run_card['nevents'])>=10000 and self.run_card['reweight_scale']=='.true.' and int(self.run_card['ickkw']) != 4:
1663
1791
                   message = message + \
1664
1792
                       ('\n      Ren. and fac. scale uncertainty: +%0.1f%% -%0.1f%%') % \
1665
1793
                       (scale_pdf_info['scale_upp'], scale_pdf_info['scale_low'])
1666
 
                if int(self.run_card['nevents'])>=10000 and self.run_card['reweight_PDF']=='.true.':
 
1794
                if int(self.run_card['nevents'])>=10000 and self.run_card['reweight_PDF']=='.true.' and int(self.run_card['ickkw']) != 4:
1667
1795
                   message = message + \
1668
1796
                       ('\n      PDF uncertainty: +%0.1f%% -%0.1f%%') % \
1669
1797
                       (scale_pdf_info['pdf_upp'], scale_pdf_info['pdf_low'])
1776
1904
             r"Quadruple precision used\:\s+(?P<nqdp>\d+).*"+\
1777
1905
             r"Initialization phase\-space points\:\s+(?P<nini>\d+).*"+\
1778
1906
             r"Unknown return code \(100\)\:\s+(?P<n100>\d+).*"+\
1779
 
             r"Unknown return code \(10\)\:\s+(?P<n10>\d+).*"+\
1780
 
             r"Unknown return code \(1\)\:\s+(?P<n1>\d+)",re.DOTALL)
 
1907
             r"Unknown return code \(10\)\:\s+(?P<n10>\d+).*",re.DOTALL)
 
1908
    
 
1909
        unit_code_meaning = { 0 : 'Not identified (CTModeRun != -1)',
 
1910
                              1 : 'CutTools (double precision)',
 
1911
                              2 : 'PJFry++',
 
1912
                              3 : 'IREGI',
 
1913
                              4 : 'Golem95',
 
1914
                              9 : 'CutTools (quadruple precision)'}
 
1915
        RetUnit_finder =re.compile(
 
1916
                           r"#Unit\s*(?P<unit>\d+)\s*=\s*(?P<n_occurences>\d+)")
 
1917
    #Unit
1781
1918
    
1782
1919
        for gv_log in log_GV_files:
1783
 
            logfile=open(gv_log,'r')            
1784
 
            UPS_stats = re.search(UPS_stat_finder,logfile.read())
1785
 
            logfile.close()
 
1920
            channel_name = '/'.join(gv_log.split('/')[-5:-1])
 
1921
            log=open(gv_log,'r').read()                
 
1922
            UPS_stats = re.search(UPS_stat_finder,log)
 
1923
            for retunit_stats in re.finditer(RetUnit_finder, log):
 
1924
                if channel_name not in stats['UPS'].keys():
 
1925
                    stats['UPS'][channel_name] = [0]*10+[[0]*10]
 
1926
                stats['UPS'][channel_name][10][int(retunit_stats.group('unit'))] \
 
1927
                                     += int(retunit_stats.group('n_occurences'))
1786
1928
            if not UPS_stats is None:
1787
 
                channel_name = '/'.join(gv_log.split('/')[-5:-1])
1788
1929
                try:
1789
1930
                    stats['UPS'][channel_name][0] += int(UPS_stats.group('ntot'))
1790
1931
                    stats['UPS'][channel_name][1] += int(UPS_stats.group('nsun'))
1796
1937
                    stats['UPS'][channel_name][7] += int(UPS_stats.group('nini'))
1797
1938
                    stats['UPS'][channel_name][8] += int(UPS_stats.group('n100'))
1798
1939
                    stats['UPS'][channel_name][9] += int(UPS_stats.group('n10'))
1799
 
                    stats['UPS'][channel_name][10] += int(UPS_stats.group('n1'))
1800
1940
                except KeyError:
1801
1941
                    stats['UPS'][channel_name] = [int(UPS_stats.group('ntot')),
1802
1942
                      int(UPS_stats.group('nsun')),int(UPS_stats.group('nsps')),
1803
1943
                      int(UPS_stats.group('nups')),int(UPS_stats.group('neps')),
1804
1944
                      int(UPS_stats.group('nddp')),int(UPS_stats.group('nqdp')),
1805
1945
                      int(UPS_stats.group('nini')),int(UPS_stats.group('n100')),
1806
 
                      int(UPS_stats.group('n10')),int(UPS_stats.group('n1'))]
 
1946
                      int(UPS_stats.group('n10')),[0]*10]
1807
1947
        debug_msg = ""
1808
1948
        if len(stats['UPS'].keys())>0:
1809
1949
            nTotPS  = sum([chan[0] for chan in stats['UPS'].values()],0)
1816
1956
            nTotini = sum([chan[7] for chan in stats['UPS'].values()],0)
1817
1957
            nTot100 = sum([chan[8] for chan in stats['UPS'].values()],0)
1818
1958
            nTot10  = sum([chan[9] for chan in stats['UPS'].values()],0)
1819
 
            nTot1  = sum([chan[10] for chan in stats['UPS'].values()],0)
 
1959
            nTot1  = [sum([chan[10][i] for chan in stats['UPS'].values()],0) \
 
1960
                                                             for i in range(10)]
1820
1961
            UPSfracs = [(chan[0] , 0.0 if chan[1][0]==0 else \
1821
1962
                 float(chan[1][4]*100)/chan[1][0]) for chan in stats['UPS'].items()]
1822
1963
            maxUPS = max(UPSfracs, key = lambda w: w[1])
1830
1971
            tmpStr += '\n    Only double precision used:          %d'%nTotddp
1831
1972
            tmpStr += '\n    Quadruple precision used:            %d'%nTotqdp
1832
1973
            tmpStr += '\n    Initialization phase-space points:   %d'%nTotini
 
1974
            tmpStr += '\n    Reduction methods used:'
 
1975
            red_methods = [(unit_code_meaning[i],nTot1[i]) for i in \
 
1976
                                         unit_code_meaning.keys() if nTot1[i]>0]
 
1977
            for method, n in sorted(red_methods, key= lambda l: l[1], reverse=True):
 
1978
                tmpStr += '\n      > %s%s%s'%(method,' '*(33-len(method)),n)                
1833
1979
            if nTot100 != 0:
1834
1980
                debug_msg += '\n  Unknown return code (100):             %d'%nTot100
1835
1981
            if nTot10 != 0:
1836
1982
                debug_msg += '\n  Unknown return code (10):              %d'%nTot10
1837
 
            if nTot1 != 0:
1838
 
                debug_msg += '\n  Unknown return code (1):               %d'%nTot1
 
1983
            nUnknownUnit = sum(nTot1[u] for u in range(10) if u \
 
1984
                                                not in unit_code_meaning.keys())
 
1985
            if nUnknownUnit != 0:
 
1986
                debug_msg += '\n  Unknown return code (1):               %d'\
 
1987
                                                                   %nUnknownUnit
1839
1988
 
1840
1989
            if maxUPS[1]>0.001:
1841
1990
                message += tmpStr
2138
2287
        Event dir. Return the name of the event file created
2139
2288
        """
2140
2289
        scale_pdf_info={}
2141
 
        if self.run_card['reweight_scale'] == '.true.' or self.run_card['reweight_PDF'] == '.true.':
 
2290
        if (self.run_card['reweight_scale'] == '.true.' or self.run_card['reweight_PDF'] == '.true.') and int(self.run_card['ickkw']) != 4 :
2142
2291
            scale_pdf_info = self.run_reweight(options['reweightonly'])
2143
2292
 
2144
2293
        self.update_status('Collecting events', level='parton', update_results=True)
2147
2296
        p = misc.Popen(['./collect_events'], cwd=pjoin(self.me_dir, 'SubProcesses'),
2148
2297
                stdin=subprocess.PIPE, 
2149
2298
                stdout=open(pjoin(self.me_dir, 'collect_events.log'), 'w'))
2150
 
        if event_norm == 'sum':
 
2299
        if event_norm.lower() == 'sum':
2151
2300
            p.communicate(input = '1\n')
 
2301
        elif event_norm.lower() == 'unity':
 
2302
            p.communicate(input = '3\n')
2152
2303
        else:
2153
2304
            p.communicate(input = '2\n')
2154
2305
 
2158
2309
        if not os.path.exists(pjoin(self.me_dir, 'SubProcesses', filename)):
2159
2310
            raise aMCatNLOError('An error occurred during event generation. ' + \
2160
2311
                    'The event file has not been created. Check collect_events.log')
2161
 
        evt_file = pjoin(self.me_dir, 'Events', self.run_name, 'events.lhe')
2162
 
        files.mv(pjoin(self.me_dir, 'SubProcesses', filename), evt_file)
2163
 
        misc.call(['gzip', evt_file])
 
2312
        evt_file = pjoin(self.me_dir, 'Events', self.run_name, 'events.lhe.gz')
 
2313
        misc.gzip(pjoin(self.me_dir, 'SubProcesses', filename), stdout=evt_file)
2164
2314
        if not options['reweightonly']:
2165
2315
            self.print_summary(options, 2, mode, scale_pdf_info)
2166
 
        logger.info('The %s.gz file has been generated.\n' \
2167
 
                % (evt_file))
 
2316
        logger.info('The %s file has been generated.\n' % (evt_file))
2168
2317
        self.results.add_detail('nb_event', nevents)
2169
2318
        self.update_status('Events generated', level='parton', update_results=True)
2170
 
        return evt_file
 
2319
        return evt_file[:-3]
2171
2320
 
2172
2321
 
2173
2322
    def run_mcatnlo(self, evt_file):
2174
2323
        """runs mcatnlo on the generated event file, to produce showered-events
2175
2324
        """
2176
2325
        logger.info('Prepairing MCatNLO run')
2177
 
        try:
2178
 
            misc.call(['gunzip', evt_file])
 
2326
        try:     
 
2327
            misc.gunzip(evt_file)
2179
2328
        except Exception:
2180
2329
            pass
2181
2330
 
2400
2549
                raise aMCatNLOError('No file has been generated, an error occurred.'+\
2401
2550
             ' More information in %s' % pjoin(os.getcwd(), 'amcatnlo_run.log'))
2402
2551
 
 
2552
            # run the plot creation in a secure way
 
2553
            try:
 
2554
                self.do_plot('%s -f' % self.run_name)
 
2555
            except Exception, error:
 
2556
                logger.info("Fail to make the plot. Continue...")
 
2557
                pass
 
2558
 
2403
2559
        elif out_id == 'TOP':
2404
2560
            #copy the topdrawer file(s) back in events
2405
2561
            topfiles = []
2528
2684
            shutil.rmtree(pjoin(run_dir_path,'RunMaterial'))
2529
2685
        # end of the run, gzip files and print out the message/warning
2530
2686
        for f in to_gzip:
2531
 
            misc.call(['gzip', f])
 
2687
            misc.gzip(f)
2532
2688
        if message:
2533
2689
            logger.info(message)
2534
2690
        if warning:
2642
2798
            return 
2643
2799
        
2644
2800
        tag = self.run_card['run_tag']
2645
 
#        if 'pythia' in self.to_store:
2646
 
#            self.update_status('Storing Pythia files of Previous run', level='pythia', error=True)
2647
 
#            os.system('mv -f %(path)s/pythia_events.hep %(path)s/%(name)s/%(tag)s_pythia_events.hep' % 
2648
 
#                  {'name': self.run_name, 'path' : pjoin(self.me_dir,'Events'),
2649
 
#                   'tag':tag})
2650
 
#            os.system('gzip -f %s/%s_pythia_events.hep' % ( 
2651
 
#                                pjoin(self.me_dir,'Events',self.run_name), tag))
2652
 
#            self.to_store.remove('pythia')
2653
 
#            self.update_status('Done', level='pythia',makehtml=False,error=True)
2654
2801
        
2655
2802
        self.to_store = []
2656
2803
 
2719
2866
        content += 'PDLABEL=%s\n' % pdlabel
2720
2867
        content += 'ALPHAEW=%s\n' % self.banner.get_detail('param_card', 'sminputs', 1).value
2721
2868
        #content += 'PDFSET=%s\n' % self.banner.get_detail('run_card', 'lhaid')
2722
 
        content += 'PDFSET=%s\n' % max([init_dict['pdfsup1'],init_dict['pdfsup2']])
 
2869
        #content += 'PDFSET=%s\n' % max([init_dict['pdfsup1'],init_dict['pdfsup2']])
2723
2870
        content += 'TMASS=%s\n' % self.banner.get_detail('param_card', 'mass', 6).value
2724
2871
        content += 'TWIDTH=%s\n' % self.banner.get_detail('param_card', 'decay', 6).value
2725
2872
        content += 'ZMASS=%s\n' % self.banner.get_detail('param_card', 'mass', 23).value
2759
2906
            content += 'TAUMASS=%s\n' % new_mcmass_dict['mcmass(15)']
2760
2907
 
2761
2908
        content += 'GMASS=%s\n' % mcmass_dict[21]
2762
 
        content += 'EVENT_NORM=%s\n' % self.banner.get_detail('run_card', 'event_norm')
 
2909
        content += 'EVENT_NORM=%s\n' % self.banner.get_detail('run_card', 'event_norm').lower()
2763
2910
        # check if need to link lhapdf
2764
 
        if pdlabel == 'lhapdf':
 
2911
        if int(self.shower_card['pdfcode']) > 1 or \
 
2912
            (pdlabel=='lhapdf' and int(self.shower_card['pdfcode'])==1): 
 
2913
            # Use LHAPDF (should be correctly installed, because
 
2914
            # either events were already generated with them, or the
 
2915
            # user explicitly gives an LHAPDF number in the
 
2916
            # shower_card).
2765
2917
            self.link_lhapdf(pjoin(self.me_dir, 'lib'))
2766
2918
            lhapdfpath = subprocess.Popen([self.options['lhapdf'], '--prefix'], 
2767
 
                          stdout = subprocess.PIPE).stdout.read().strip()
 
2919
                                          stdout = subprocess.PIPE).stdout.read().strip()
2768
2920
            content += 'LHAPDFPATH=%s\n' % lhapdfpath
2769
2921
            pdfsetsdir = self.get_lhapdf_pdfsetsdir()
2770
 
            lhaid_list = [max([init_dict['pdfsup1'],init_dict['pdfsup2']])]
 
2922
            if self.shower_card['pdfcode']==1:
 
2923
                lhaid_list = [max([init_dict['pdfsup1'],init_dict['pdfsup2']])]
 
2924
                content += 'PDFCODE=%s\n' % max([init_dict['pdfsup1'],init_dict['pdfsup2']])
 
2925
            else:
 
2926
                lhaid_list = [abs(int(self.shower_card['pdfcode']))]
 
2927
                content += 'PDFCODE=%s\n' % self.shower_card['pdfcode']
2771
2928
            self.copy_lhapdf_set(lhaid_list, pdfsetsdir)
 
2929
        elif int(self.shower_card['pdfcode'])==1:
 
2930
            # Try to use LHAPDF because user wants to use the same PDF
 
2931
            # as was used for the event generation. However, for the
 
2932
            # event generation, LHAPDF was not used, so non-trivial to
 
2933
            # see if if LHAPDF is available with the corresponding PDF
 
2934
            # set. If not found, give a warning and use build-in PDF
 
2935
            # set instead.
 
2936
            try:
 
2937
                lhapdfpath = subprocess.Popen([self.options['lhapdf'], '--prefix'], 
 
2938
                                              stdout = subprocess.PIPE).stdout.read().strip()
 
2939
                self.link_lhapdf(pjoin(self.me_dir, 'lib'))
 
2940
                content += 'LHAPDFPATH=%s\n' % lhapdfpath
 
2941
                pdfsetsdir = self.get_lhapdf_pdfsetsdir()
 
2942
                lhaid_list = [max([init_dict['pdfsup1'],init_dict['pdfsup2']])]
 
2943
                content += 'PDFCODE=%s\n' % max([init_dict['pdfsup1'],init_dict['pdfsup2']])
 
2944
                self.copy_lhapdf_set(lhaid_list, pdfsetsdir)
 
2945
            except Exception:
 
2946
                logger.warning('Trying to shower events using the same PDF in the shower as used in the generation'+\
 
2947
                                   ' of the events using LHAPDF. However, no valid LHAPDF installation found with the'+\
 
2948
                                   ' needed PDF set. Will use default internal PDF for the shower instead. To use the'+\
 
2949
                                   ' same set as was used in the event generation install LHAPDF and set the path using'+\
 
2950
                                   ' "set /path_to_lhapdf/bin/lhapdf-config" from the MadGraph5_aMC@NLO python shell')
 
2951
                content += 'LHAPDFPATH=\n' 
 
2952
                content += 'PDFCODE=0\n'
2772
2953
        else:
2773
 
            #overwrite the PDFCODE variable in order to use internal pdf
2774
2954
            content += 'LHAPDFPATH=\n' 
2775
2955
            content += 'PDFCODE=0\n'
2776
2956
 
 
2957
        content += 'ICKKW=%s\n' % self.banner.get_detail('run_card', 'ickkw')
 
2958
        content += 'PTJCUT=%s\n' % self.banner.get_detail('run_card', 'ptj')
2777
2959
        # add the pythia8/hwpp path(s)
2778
2960
        if self.options['pythia8_path']:
2779
2961
            content+='PY8PATH=%s\n' % self.options['pythia8_path']
2942
3124
        """runs the jobs in job_dict (organized as folder: [job_list]), with arguments args"""
2943
3125
        njob_split = 0
2944
3126
        self.ijob = 0
2945
 
        if self.cluster_mode == 0:
2946
 
            self.update_status((self.njobs - 1, 1, 0, run_type), level='parton')
2947
3127
 
2948
3128
        #  this is to keep track, if splitting evt generation, of the various 
2949
3129
        # folders/args in order to resubmit the jobs if some of them fail
3020
3200
        # then open the nevents_unweighted_splitted file and look for the 
3021
3201
        # number of splittings to be done
3022
3202
        nevents_file = open(pjoin(self.me_dir, 'SubProcesses', 'nevents_unweighted_splitted')).read()
 
3203
        # This skips the channels with zero events, because they are
 
3204
        # not of the form GFXX_YY, but simply GFXX
3023
3205
        pattern = re.compile(r"%s_(\d+)/events.lhe" % \
3024
3206
                          pjoin(pdir, 'G%s%s' % (arg,channel)))
3025
3207
        matches = re.findall(pattern, nevents_file)
3150
3332
                     pjoin(self.me_dir, 'SubProcesses', 'randinit'),
3151
3333
                     pjoin(cwd, 'symfact.dat'),
3152
3334
                     pjoin(cwd, 'iproc.dat'),
 
3335
                     pjoin(cwd, 'initial_states_map.dat'),
3153
3336
                     pjoin(cwd, 'param_card.dat'),
3154
3337
                     pjoin(cwd, 'FKS_params.dat')]
3155
3338
 
3159
3342
        if os.path.exists(pjoin(self.me_dir,'SubProcesses','OLE_order.olc')):
3160
3343
            input_files.append(pjoin(cwd, 'OLE_order.olc'))
3161
3344
 
3162
 
        # LHAPDF dynamic libraries (needed for lhapdf6)
3163
 
        lhalibs = ['libLHAPDF.dylib', 'libLHAPDF.so'] 
3164
 
        for lib in [pjoin(self.me_dir, 'lib', l) for l in lhalibs \
3165
 
           if os.path.exists(pjoin(self.me_dir, 'lib', l))]:
3166
 
            input_files.append(lib)
3167
 
      
3168
3345
        # File for the loop (might not be present if MadLoop is not used)
3169
 
        if os.path.exists(pjoin(cwd, 'MadLoopParams.dat')):
3170
 
            to_add = ['MadLoopParams.dat', 'ColorDenomFactors.dat', 
3171
 
                                   'ColorNumFactors.dat','HelConfigs.dat']
3172
 
            for name in to_add:
3173
 
                input_files.append(pjoin(cwd, name))
3174
 
 
3175
 
                to_check = ['HelFilter.dat','LoopFilter.dat']
3176
 
            for name in to_check:
3177
 
                if os.path.exists(pjoin(cwd, name)):
3178
 
                    input_files.append(pjoin(cwd, name))
 
3346
        if os.path.exists(pjoin(cwd,'MadLoop5_resources')):
 
3347
            input_files.append(pjoin(cwd, 'MadLoop5_resources.tar.gz'))
 
3348
            if not os.path.exists(pjoin(cwd,'MadLoop5_resources.tar.gz')):
 
3349
                tf=tarfile.open(pjoin(cwd,'MadLoop5_resources.tar.gz'),'w:gz',
 
3350
                                                                 dereference=True)
 
3351
                tf.add(pjoin(cwd,'MadLoop5_resources'),arcname='MadLoop5_resources')
 
3352
                tf.close()
3179
3353
 
3180
3354
        Ire = re.compile("for i in ([\d\s]*) ; do")
3181
3355
        try : 
3206
3380
                        to_move = ['mint_grids', 'grid.MC_integer']
3207
3381
                    else: 
3208
3382
                        to_move  = []
 
3383
                    if self.run_card['iappl'] =='2':
 
3384
                        for grid in glob.glob(pjoin(cwd,base,'grid_obs_*_in.root')):
 
3385
                            to_move.append(grid)
3209
3386
                    if not os.path.exists(pjoin(cwd,current)):
3210
3387
                        os.mkdir(pjoin(cwd,current))
3211
3388
                        input_files.append(pjoin(cwd, current))
3321
3498
        self.banner.write(pjoin(self.me_dir, 'Events', self.run_name, 
3322
3499
                          '%s_%s_banner.txt' % (self.run_name, self.run_tag)))
3323
3500
 
3324
 
        self.get_characteristics(pjoin(self.me_dir, 'SubProcesses', 'proc_characteristics'))
 
3501
        self.get_characteristics(pjoin(self.me_dir, 
 
3502
                                        'SubProcesses', 'proc_characteristics'))
3325
3503
 
3326
3504
        #define a bunch of log files
3327
3505
        amcatnlo_log = pjoin(self.me_dir, 'compile_amcatnlo.log')
3388
3566
            except KeyError:
3389
3567
                pass
3390
3568
 
 
3569
        # read the run_card to find if applgrid is used or not
 
3570
        if self.run_card['iappl'] != '0':
 
3571
            os.environ['applgrid'] = 'True'
 
3572
            # check versions of applgrid and amcfast
 
3573
            for code in ['applgrid','amcfast']:
 
3574
                try:
 
3575
                    p = subprocess.Popen([self.options[code], '--version'], \
 
3576
                                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
3577
                    output, error = p.communicate()
 
3578
                    if code is 'applgrid' and output < '1.4.63':
 
3579
                        raise aMCatNLOError('Version of APPLgrid is too old. Use 1.4.69 or later.'\
 
3580
                                                +' You are using %s',output)
 
3581
                    if code is 'amcfast' and output < '1.1.1':
 
3582
                        raise aMCatNLOError('Version of aMCfast is too old. Use 1.1.1 or later.'\
 
3583
                                                +' You are using %s',output)
 
3584
                except Exception:
 
3585
                    raise aMCatNLOError(('No valid %s installation found. \n' + \
 
3586
                          'Please set the path to %s-config by using \n' + \
 
3587
                          'MG5_aMC> set <absolute-path-to-%s>/bin/%s-config \n') % (code,code,code,code))
 
3588
            # set-up the Source/make_opts with the correct applgrid-config file
 
3589
            appllibs="  APPLLIBS=$(shell %s --ldcflags) $(shell %s --ldflags) \n" \
 
3590
                             % (self.options['applgrid'],self.options['amcfast'])
 
3591
            text=open(pjoin(self.me_dir,'Source','make_opts'),'r').readlines()
 
3592
            text_out=[]
 
3593
            for line in text:
 
3594
                if line.strip().startswith('APPLLIBS=$'):
 
3595
                    line=appllibs
 
3596
                text_out.append(line)
 
3597
            open(pjoin(self.me_dir,'Source','make_opts'),'w').writelines(text_out)
 
3598
        else:
 
3599
            try:
 
3600
                del os.environ['applgrid']
 
3601
            except KeyError:
 
3602
                pass
 
3603
 
3391
3604
        try: 
3392
3605
            os.environ['fastjet_config'] = self.options['fastjet']
3393
3606
        except (TypeError, KeyError):
3455
3668
                 " seems to have been compiled with a different compiler than"+\
3456
3669
                    " the one specified in MG5_aMC. Please recompile CutTools.")
3457
3670
 
3458
 
        # check if virtuals have been generated
3459
 
        proc_card = open(pjoin(self.me_dir, 'Cards', 'proc_card_mg5.dat')).read()
 
3671
        # make IREGI (only necessary with MG option output_dependencies='internal')
 
3672
        if not os.path.exists(os.path.realpath(pjoin(libdir, 'libiregi.a'))) \
 
3673
           and os.path.exists(pjoin(sourcedir,'IREGI')):
 
3674
                logger.info('Compiling IREGI (can take a couple of minutes) ...')
 
3675
                misc.compile(['IREGI'], cwd = sourcedir)
 
3676
                logger.info('          ...done.')
 
3677
 
 
3678
        if os.path.exists(pjoin(libdir, 'libiregi.a')):
 
3679
            # Verify compatibility between current compiler and the one which was
 
3680
            # used when last compiling IREGI (if specified).
 
3681
            compiler_log_path = pjoin(os.path.dirname((os.path.realpath(pjoin(
 
3682
                                libdir, 'libiregi.a')))),'compiler_version.log')
 
3683
            if os.path.exists(compiler_log_path):
 
3684
                compiler_version_used = open(compiler_log_path,'r').read()
 
3685
                if not str(misc.get_gfortran_version(misc.detect_current_compiler(\
 
3686
                       pjoin(sourcedir,'make_opts')))) in compiler_version_used:
 
3687
                    if os.path.exists(pjoin(sourcedir,'IREGI')):
 
3688
                        logger.info('IREGI was compiled with a different fortran'+\
 
3689
                                            ' compiler. Re-compiling it now...')
 
3690
                        misc.compile(['cleanIR'], cwd = sourcedir)
 
3691
                        misc.compile(['IREGI'], cwd = sourcedir)
 
3692
                        logger.info('          ...done.')
 
3693
                    else:
 
3694
                        raise aMCatNLOError("IREGI installation in %s"\
 
3695
                                %os.path.realpath(pjoin(libdir, 'libiregi.a'))+\
 
3696
                 " seems to have been compiled with a different compiler than"+\
 
3697
                    " the one specified in MG5_aMC. Please recompile IREGI.")
 
3698
 
 
3699
        # check if MadLoop virtuals have been generated
3460
3700
        if self.proc_characteristics['has_loops'].lower() == 'true' and \
3461
 
           mode in ['NLO', 'aMC@NLO', 'noshower']:
3462
 
            tests.append('check_poles')
 
3701
                          not os.path.exists(pjoin(self.me_dir,'OLP_virtuals')):
 
3702
            os.environ['madloop'] = 'true'
 
3703
            if mode in ['NLO', 'aMC@NLO', 'noshower']:
 
3704
                tests.append('check_poles')
 
3705
        else:
 
3706
            os.unsetenv('madloop')
3463
3707
 
3464
3708
        # make and run tests (if asked for), gensym and make madevent in each dir
3465
3709
        self.update_status('Compiling directories...', level=None)
3831
4075
                if self.run_card['parton_shower'].upper() == 'PYTHIA6Q':
3832
4076
                    logger.error("""FxFx merging does not work with Q-squared ordered showers.""")
3833
4077
                    raise self.InvalidCmd(error)
3834
 
                elif self.run_card['parton_shower'].upper() != 'HERWIG6':
 
4078
                elif self.run_card['parton_shower'].upper() != 'HERWIG6' and self.run_card['parton_shower'].upper() != 'PYTHIA8':
3835
4079
                    question="FxFx merging not tested for %s shower. Do you want to continue?\n"  % self.run_card['parton_shower'] + \
3836
4080
                        "Type \'n\' to stop or \'y\' to continue"
3837
4081
                    answers = ['n','y']
3896
4140
                            "the last available results")
3897
4141
_launch_parser.add_option("-n", "--name", default=False, dest='run_name',
3898
4142
                            help="Provide a name to the run")
 
4143
_launch_parser.add_option("-a", "--appl_start_grid", default=False, dest='appl_start_grid',
 
4144
                            help="For use with APPLgrid only: start from existing grids")
3899
4145
 
3900
4146
 
3901
4147
_generate_events_usage = "generate_events [MODE] [options]\n" + \
3946
4192
                            help="Skip compilation. Ignored if no executable is found")
3947
4193
_calculate_xsect_parser.add_option("-n", "--name", default=False, dest='run_name',
3948
4194
                            help="Provide a name to the run")
 
4195
_calculate_xsect_parser.add_option("-a", "--appl_start_grid", default=False, dest='appl_start_grid',
 
4196
                            help="For use with APPLgrid only: start from existing grids")
 
4197
_calculate_xsect_parser.add_option("-o", "--only_generation", default=False, action='store_true',
 
4198
                            help="Skip grid set up, just generate events starting from " + \
 
4199
                            "the last available results")
3949
4200
 
3950
4201
_shower_usage = 'shower run_name [options]\n' + \
3951
4202
        '-- do shower/hadronization on parton-level file generated for run run_name\n' + \