← Back to branch summary

~madteam/mg5amcnlo/series2.0

« back to all changes in this revision

Viewing changes to madgraph/interface/madevent_interface.py

merge with branch ReadOnly gridpack

Show diffs side-by-side

added added

removed removed

Lines of Context:
madgraph/interface/madevent_interface.py
18
18
from __future__ import division
19
19
 
20
20
import collections
 
21
import itertools
21
22
import glob
22
23
import logging
23
24
import math
3533
3534
 
3534
3535
        
3535
3536
 
3536
 
        if self.run_card['gridpack']: #not hasattr(self, "refine_mode") or self.refine_mode == "old":
3537
 
            try:
3538
 
                os.remove(pjoin(self.me_dir,'SubProcesses', 'combine.log'))
3539
 
            except Exception:
3540
 
                pass
3541
 
 
3542
 
            cluster.onecore.launch_and_wait('../bin/internal/run_combine', 
3543
 
                                       args=[self.run_name],
3544
 
                                       cwd=pjoin(self.me_dir,'SubProcesses'),
3545
 
                                       stdout=pjoin(self.me_dir,'SubProcesses', 'combine.log'),
3546
 
                                       required_output=[pjoin(self.me_dir,'SubProcesses', 'combine.log')])
3547
 
 
3548
 
            #self.cluster.launch_and_wait('../bin/internal/run_combine', 
3549
 
            #                                cwd=pjoin(self.me_dir,'SubProcesses'),
3550
 
            #                                stdout=pjoin(self.me_dir,'SubProcesses', 'combine.log'),
3551
 
            #                                required_output=[pjoin(self.me_dir,'SubProcesses', 'combine.log')])
 
3537
        if self.run_card['gridpack'] and isinstance(self, GridPackCmd):
 
3538
            return GridPackCmd.do_combine_events(self, line)
 
3539
    
 
3540
        # Define The Banner
 
3541
        tag = self.run_card['run_tag']
 
3542
        # Update the banner with the pythia card
 
3543
        if not self.banner:
 
3544
            self.banner = banner_mod.recover_banner(self.results, 'parton')
 
3545
        self.banner.load_basic(self.me_dir)
 
3546
        # Add cross-section/event information
 
3547
        self.banner.add_generation_info(self.results.current['cross'], self.run_card['nevents'])
 
3548
        if not hasattr(self, 'random_orig'): self.random_orig = 0
 
3549
        self.banner.change_seed(self.random_orig)
 
3550
        if not os.path.exists(pjoin(self.me_dir, 'Events', self.run_name)):
 
3551
            os.mkdir(pjoin(self.me_dir, 'Events', self.run_name))
 
3552
        self.banner.write(pjoin(self.me_dir, 'Events', self.run_name, 
 
3553
                                '%s_%s_banner.txt' % (self.run_name, tag)))
 
3554
        
 
3555
        
 
3556
        get_wgt = lambda event: event.wgt            
 
3557
        AllEvent = lhe_parser.MultiEventFile()
 
3558
        AllEvent.banner = self.banner
 
3559
        
 
3560
        partials = 0 # if too many file make some partial unweighting
 
3561
        sum_xsec, sum_xerru, sum_axsec = 0,[],0
 
3562
        for Gdir in self.get_Gdir():
 
3563
            if os.path.exists(pjoin(Gdir, 'events.lhe')):
 
3564
                result = sum_html.OneResult('')
 
3565
                result.read_results(pjoin(Gdir, 'results.dat'))
 
3566
                AllEvent.add(pjoin(Gdir, 'events.lhe'), 
 
3567
                             result.get('xsec'),
 
3568
                             result.get('xerru'),
 
3569
                             result.get('axsec')
 
3570
                             )
 
3571
 
 
3572
                sum_xsec += result.get('xsec')
 
3573
                sum_xerru.append(result.get('xerru'))
 
3574
                sum_axsec += result.get('axsec')
 
3575
                
 
3576
                if len(AllEvent) >= 80: #perform a partial unweighting
 
3577
                    AllEvent.unweight(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % partials),
 
3578
                          get_wgt, log_level=5,  trunc_error=1e-2, event_target=self.run_card['nevents'])
 
3579
                    AllEvent = lhe_parser.MultiEventFile()
 
3580
                    AllEvent.banner = self.banner
 
3581
                    AllEvent.add(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % partials),
 
3582
                                 sum_xsec,
 
3583
                                 math.sqrt(sum(x**2 for x in sum_xerru)),
 
3584
                                 sum_axsec) 
 
3585
                    partials +=1
 
3586
        
 
3587
        if not hasattr(self,'proc_characteristic'):
 
3588
            self.proc_characteristic = self.get_characteristics()
3552
3589
            
3553
 
            output = misc.mult_try_open(pjoin(self.me_dir,'SubProcesses','combine.log')).read()
3554
 
            # Store the number of unweighted events for the results object
3555
 
            pat = re.compile(r'''\s*Unweighting\s*selected\s*(\d+)\s*events''')
3556
 
            try:      
3557
 
                nb_event = pat.search(output).groups()[0]
3558
 
            except AttributeError:
3559
 
                time.sleep(10)
3560
 
                output = misc.mult_try_open(pjoin(self.me_dir,'SubProcesses','combine.log')).read()
 
3590
        nb_event = AllEvent.unweight(pjoin(self.me_dir, "Events", self.run_name, "unweighted_events.lhe.gz"),
 
3591
                          get_wgt, trunc_error=1e-2, event_target=self.run_card['nevents'],
 
3592
                          log_level=logging.DEBUG, normalization=self.run_card['event_norm'],
 
3593
                          proc_charac=self.proc_characteristic)
 
3594
        
 
3595
        if partials:
 
3596
            for i in range(partials):
3561
3597
                try:
3562
 
                    nb_event = pat.search(output).groups()[0]
3563
 
                except AttributeError:
3564
 
                    logger.warning('Fail to read the number of unweighted events in the combine.log file')
3565
 
                    nb_event = 0
3566
 
 
3567
 
            self.results.add_detail('nb_event', nb_event)
3568
 
            
3569
 
            
3570
 
            # Define The Banner
3571
 
            tag = self.run_card['run_tag']
3572
 
            
3573
 
            # Update the banner with the pythia card
3574
 
            if not self.banner:
3575
 
                self.banner = banner_mod.recover_banner(self.results, 'parton')
3576
 
            self.banner.load_basic(self.me_dir)
3577
 
            # Add cross-section/event information
3578
 
            self.banner.add_generation_info(self.results.current['cross'], nb_event)
3579
 
            if not hasattr(self, 'random_orig'): self.random_orig = 0
3580
 
            self.banner.change_seed(self.random_orig)
3581
 
            if not os.path.exists(pjoin(self.me_dir, 'Events', self.run_name)):
3582
 
                os.mkdir(pjoin(self.me_dir, 'Events', self.run_name))
3583
 
            self.banner.write(pjoin(self.me_dir, 'Events', self.run_name, 
3584
 
                                         '%s_%s_banner.txt' % (self.run_name, tag)))
3585
 
            
3586
 
            
3587
 
            self.banner.add_to_file(pjoin(self.me_dir,'Events', 'events.lhe'),
3588
 
                                    out=pjoin(self.me_dir,'Events', self.run_name, 'events.lhe'))
3589
 
            self.banner.add_to_file(pjoin(self.me_dir,'Events', 'unweighted_events.lhe'),        
3590
 
                                    out=pjoin(self.me_dir,'Events', self.run_name, 'unweighted_events.lhe'))        
3591
 
 
3592
 
        else:
3593
 
            # Define The Banner
3594
 
            tag = self.run_card['run_tag']
3595
 
            # Update the banner with the pythia card
3596
 
            if not self.banner:
3597
 
                self.banner = banner_mod.recover_banner(self.results, 'parton')
3598
 
            self.banner.load_basic(self.me_dir)
3599
 
            # Add cross-section/event information
3600
 
            self.banner.add_generation_info(self.results.current['cross'], self.run_card['nevents'])
3601
 
            if not hasattr(self, 'random_orig'): self.random_orig = 0
3602
 
            self.banner.change_seed(self.random_orig)
3603
 
            if not os.path.exists(pjoin(self.me_dir, 'Events', self.run_name)):
3604
 
                os.mkdir(pjoin(self.me_dir, 'Events', self.run_name))
3605
 
            self.banner.write(pjoin(self.me_dir, 'Events', self.run_name, 
3606
 
                                    '%s_%s_banner.txt' % (self.run_name, tag)))
3607
 
            
3608
 
            
3609
 
            get_wgt = lambda event: event.wgt            
3610
 
            AllEvent = lhe_parser.MultiEventFile()
3611
 
            AllEvent.banner = self.banner
3612
 
            
3613
 
            partials = 0 # if too many file make some partial unweighting
3614
 
            sum_xsec, sum_xerru, sum_axsec = 0,[],0
3615
 
            for Gdir,mfactor in self.get_Gdir():
3616
 
                if os.path.exists(pjoin(Gdir, 'events.lhe')):
3617
 
                    result = sum_html.OneResult('')
3618
 
                    result.read_results(pjoin(Gdir, 'results.dat'))
3619
 
                    AllEvent.add(pjoin(Gdir, 'events.lhe'), 
3620
 
                                 result.get('xsec'),
3621
 
                                 result.get('xerru'),
3622
 
                                 result.get('axsec')
3623
 
                                 )
3624
 
 
3625
 
                    sum_xsec += result.get('xsec')
3626
 
                    sum_xerru.append(result.get('xerru'))
3627
 
                    sum_axsec += result.get('axsec')
3628
 
                    
3629
 
                    if len(AllEvent) >= 80: #perform a partial unweighting
3630
 
                        AllEvent.unweight(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % partials),
3631
 
                              get_wgt, log_level=5,  trunc_error=1e-2, event_target=self.run_card['nevents'])
3632
 
                        AllEvent = lhe_parser.MultiEventFile()
3633
 
                        AllEvent.banner = self.banner
3634
 
                        AllEvent.add(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % partials),
3635
 
                                     sum_xsec,
3636
 
                                     math.sqrt(sum(x**2 for x in sum_xerru)),
3637
 
                                     sum_axsec) 
3638
 
                        partials +=1
3639
 
            
3640
 
            if not hasattr(self,'proc_characteristic'):
3641
 
                self.proc_characteristic = self.get_characteristics()
3642
 
                
3643
 
            nb_event = AllEvent.unweight(pjoin(self.me_dir, "Events", self.run_name, "unweighted_events.lhe.gz"),
3644
 
                              get_wgt, trunc_error=1e-2, event_target=self.run_card['nevents'],
3645
 
                              log_level=logging.DEBUG, normalization=self.run_card['event_norm'],
3646
 
                              proc_charac=self.proc_characteristic)
3647
 
            
3648
 
            if partials:
3649
 
                for i in range(partials):
3650
 
                    try:
3651
 
                        os.remove(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % i))
3652
 
                    except Exception:
3653
 
                        os.remove(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe" % i))
3654
 
                       
3655
 
            self.results.add_detail('nb_event', nb_event)
3656
 
        
 
3598
                    os.remove(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % i))
 
3599
                except Exception:
 
3600
                    os.remove(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe" % i))
 
3601
                   
 
3602
        self.results.add_detail('nb_event', nb_event)
 
3603
    
3657
3604
        if self.run_card['bias_module'].lower() not in  ['dummy', 'none']:
3658
3605
            self.correct_bias()
3659
3606
        
3757
3704
                           "    cd Subprocesses; ../bin/internal/combine_events\n"+
3758
3705
                           "  to have your events if those one are missing.")
3759
3706
        else:
3760
 
            for P_path in SubProcesses.get_subP(self.me_dir):
3761
 
                G_dir = [G for G in os.listdir(P_path) if G.startswith('G') and 
3762
 
                                                    os.path.isdir(pjoin(P_path,G))]
3763
 
                for G in G_dir:
3764
 
                    G_path = pjoin(P_path,G)
3765
 
                    try:
3766
 
                        # Remove events file (if present)
3767
 
                        if os.path.exists(pjoin(G_path, 'events.lhe')):
3768
 
                            os.remove(pjoin(G_path, 'events.lhe'))
3769
 
                    except Exception:
3770
 
                        continue
3771
 
                    #try:
3772
 
                    #    # Store results.dat
3773
 
                    #    if os.path.exists(pjoin(G_path, 'results.dat')):
3774
 
                    #        input = pjoin(G_path, 'results.dat')
3775
 
                    #        output = pjoin(G_path, '%s_results.dat' % run)
3776
 
                    #        files.cp(input, output) 
3777
 
                    #except Exception:
3778
 
                    #    continue                    
3779
 
                    # Store log
3780
 
                    try:
3781
 
                        if os.path.exists(pjoin(G_path, 'log.txt')):
3782
 
                            input = pjoin(G_path, 'log.txt')
3783
 
                            output = pjoin(G_path, '%s_log.txt' % run)
3784
 
                            files.mv(input, output) 
3785
 
                    except Exception:
3786
 
                        continue
3787
 
                    #try:   
3788
 
                    #    # Grid
3789
 
                    #    for name in ['ftn26']:
3790
 
                    #        if os.path.exists(pjoin(G_path, name)):
3791
 
                    #            if os.path.exists(pjoin(G_path, '%s_%s.gz'%(run,name))):
3792
 
                    #                os.remove(pjoin(G_path, '%s_%s.gz'%(run,name)))
3793
 
                    #            input = pjoin(G_path, name)
3794
 
                    #            output = pjoin(G_path, '%s_%s' % (run,name))
3795
 
                    #            files.mv(input, output)
3796
 
                    #            misc.gzip(pjoin(G_path, output), error=None) 
3797
 
                    #except Exception:
3798
 
                    #    continue
3799
 
                    # Delete ftn25 to ensure reproducible runs
3800
 
                    if os.path.exists(pjoin(G_path, 'ftn25')):
3801
 
                        os.remove(pjoin(G_path, 'ftn25'))
 
3707
            for G_path in self.get_Gdir():
 
3708
                try:
 
3709
                    # Remove events file (if present)
 
3710
                    if os.path.exists(pjoin(G_path, 'events.lhe')):
 
3711
                        os.remove(pjoin(G_path, 'events.lhe'))
 
3712
                except Exception:
 
3713
                    continue
 
3714
                #try:
 
3715
                #    # Store results.dat
 
3716
                #    if os.path.exists(pjoin(G_path, 'results.dat')):
 
3717
                #        input = pjoin(G_path, 'results.dat')
 
3718
                #        output = pjoin(G_path, '%s_results.dat' % run)
 
3719
                #        files.cp(input, output) 
 
3720
                #except Exception:
 
3721
                #    continue                    
 
3722
                # Store log
 
3723
                try:
 
3724
                    if os.path.exists(pjoin(G_path, 'log.txt')):
 
3725
                        input = pjoin(G_path, 'log.txt')
 
3726
                        output = pjoin(G_path, '%s_log.txt' % run)
 
3727
                        files.mv(input, output) 
 
3728
                except Exception:
 
3729
                    continue
 
3730
                #try:   
 
3731
                #    # Grid
 
3732
                #    for name in ['ftn26']:
 
3733
                #        if os.path.exists(pjoin(G_path, name)):
 
3734
                #            if os.path.exists(pjoin(G_path, '%s_%s.gz'%(run,name))):
 
3735
                #                os.remove(pjoin(G_path, '%s_%s.gz'%(run,name)))
 
3736
                #            input = pjoin(G_path, name)
 
3737
                #            output = pjoin(G_path, '%s_%s' % (run,name))
 
3738
                #            files.mv(input, output)
 
3739
                #            misc.gzip(pjoin(G_path, output), error=None) 
 
3740
                #except Exception:
 
3741
                #    continue
 
3742
                # Delete ftn25 to ensure reproducible runs
 
3743
                if os.path.exists(pjoin(G_path, 'ftn25')):
 
3744
                    os.remove(pjoin(G_path, 'ftn25'))
3802
3745
 
3803
3746
        # 3) Update the index.html
3804
3747
        self.gen_card_html()
5721
5664
        else:
5722
5665
            return default
5723
5666
 
5724
 
    ############################################################################
5725
 
    def get_Pdir(self):
5726
 
        """get the list of Pdirectory if not yet saved."""
5727
 
        
5728
 
        if hasattr(self, "Pdirs"):
5729
 
            if self.me_dir in self.Pdirs[0]:
5730
 
                return self.Pdirs
5731
 
        self.Pdirs = [pjoin(self.me_dir, 'SubProcesses', l.strip()) 
5732
 
                     for l in open(pjoin(self.me_dir,'SubProcesses', 'subproc.mg'))]
5733
 
        return self.Pdirs
5734
 
        
5735
 
    ############################################################################
5736
 
    def get_Gdir(self):
 
5667
 
 
5668
        
 
5669
    ############################################################################
 
5670
    def get_Gdir(self, Pdir=None, symfact=None):
5737
5671
        """get the list of Gdirectory if not yet saved."""
5738
5672
        
5739
5673
        if hasattr(self, "Gdirs"):
5740
5674
            if self.me_dir in self.Gdirs[0]:
5741
 
                return self.Gdirs
 
5675
                if Pdir is None:
 
5676
                    if not symfact:
 
5677
                        return list(itertools.chain(*self.Gdirs[0].values()))
 
5678
                    else:
 
5679
                        return list(itertools.chain(*self.Gdirs[0].values())), self.Gdirs[1]
 
5680
                else:
 
5681
                    if not symfact:
 
5682
                        return self.Gdirs[0][Pdir]
 
5683
                    else:
 
5684
                        return self.Gdirs[0][Pdir], self.Gdirs[1]
 
5685
 
5742
5686
 
5743
5687
        Pdirs = self.get_Pdir()
5744
 
        Gdirs = []        
 
5688
        Gdirs = {self.me_dir:[]}   
 
5689
        mfactors = {}     
5745
5690
        for P in Pdirs:
5746
 
            for line in open(pjoin(P, "symfact.dat")):
 
5691
            Gdirs[P] = []
 
5692
            #for the next line do not use P, since in readonly mode it might not have symfact
 
5693
            for line in open(pjoin(self.me_dir, 'SubProcesses',os.path.basename(P), "symfact.dat")):
5747
5694
                tag, mfactor = line.split()
5748
 
                Gdirs.append( (pjoin(P, "G%s" % tag), int(mfactor)) )
5749
 
            
5750
 
        
5751
 
        self.Gdirs = Gdirs
5752
 
        return self.Gdirs
 
5695
                if int(mfactor) > 0:
 
5696
                    Gdirs[P].append( pjoin(P, "G%s" % tag) )
 
5697
                    mfactors[pjoin(P, "G%s" % tag)] = mfactor
 
5698
        self.Gdirs = (Gdirs, mfactors)
 
5699
        return self.get_Gdir(Pdir, symfact=symfact)
5753
5700
                
5754
5701
    ############################################################################
5755
5702
    def set_run_name(self, name, tag=None, level='parton', reload_card=False,
6347
6294
class GridPackCmd(MadEventCmd):
6348
6295
    """The command for the gridpack --Those are not suppose to be use interactively--"""
6349
6296
 
6350
 
    def __init__(self, me_dir = None, nb_event=0, seed=0, *completekey, **stdin):
 
6297
    def __init__(self, me_dir = None, nb_event=0, seed=0, gran=-1, *completekey, **stdin):
6351
6298
        """Initialize the command and directly run"""
6352
6299
 
6353
6300
        # Initialize properly
6354
 
        
 
6301
        self.readonly = False
6355
6302
        MadEventCmd.__init__(self, me_dir, *completekey, **stdin)
6356
6303
        self.run_mode = 0
6357
6304
        self.random = seed
6358
6305
        self.random_orig = self.random
 
6306
        self.granularity = gran
 
6307
        
6359
6308
        self.options['automatic_html_opening'] = False
 
6309
        #write the grid_card.dat on disk
 
6310
        self.nb_event = int(nb_event)
 
6311
        self.write_gridcard(nb_event, seed, gran) # set readonly on True if needed
 
6312
        self.prepare_local_dir()                  # move to gridpack dir or create local structure
6360
6313
        # Now it's time to run!
6361
6314
        if me_dir and nb_event and seed:
6362
6315
            self.launch(nb_event, seed)
6365
6318
                  'Gridpack run failed: ' + str(me_dir) + str(nb_event) + \
6366
6319
                  str(seed)
6367
6320
 
 
6321
 
 
6322
    def update_status(self, *args, **opts):
 
6323
        return
 
6324
 
 
6325
    def load_results_db(self):
 
6326
        """load the current results status"""
 
6327
        model = self.find_model_name()
 
6328
        process = self.process # define in find_model_name
 
6329
        self.results = gen_crossxhtml.AllResults(model, process, self.me_dir)
 
6330
        self.last_mode=''
 
6331
 
 
6332
    def save_random(self):
 
6333
        """save random number in appropirate file"""
 
6334
 
 
6335
        if not self.readonly:
 
6336
            fsock = open(pjoin(self.me_dir, 'SubProcesses','randinit'),'w')
 
6337
        else:
 
6338
            fsock = open('randinit','w')
 
6339
        fsock.writelines('r=%s\n' % self.random)
 
6340
 
 
6341
    def write_RunWeb(self, me_dir):
 
6342
        try:
 
6343
            super(GridPackCmd, self).write_RunWeb(me_dir)
 
6344
        except IOError:
 
6345
            self.readonly  =True
 
6346
 
 
6347
    def write_gridcard(self, nb_event, seed, gran):
 
6348
        """write the grid_card.dat file at appropriate location"""
 
6349
        
 
6350
        # first try to write grid_card within the gridpack.
 
6351
        print "WRITE GRIDCARD", self.me_dir
 
6352
        if self.readonly:
 
6353
            if not os.path.exists('Cards'):
 
6354
                os.mkdir('Cards')
 
6355
            fsock = open('grid_card.dat','w')
 
6356
        else:
 
6357
            fsock = open(pjoin(self.me_dir, 'Cards', 'grid_card.dat'),'w')
 
6358
                
 
6359
        gridpackcard = banner_mod.GridpackCard()
 
6360
        gridpackcard['GridRun'] = True
 
6361
        gridpackcard['gevents'] = nb_event
 
6362
        gridpackcard['gseed'] = seed
 
6363
        gridpackcard['ngran'] = gran
 
6364
        
 
6365
        gridpackcard.write(fsock)
 
6366
 
 
6367
    ############################################################################
 
6368
    def get_Pdir(self):
 
6369
        """get the list of Pdirectory if not yet saved."""
 
6370
        
 
6371
        if hasattr(self, "Pdirs"):
 
6372
            if self.me_dir in self.Pdirs[0]:
 
6373
                return self.Pdirs
 
6374
            
 
6375
        if not self.readonly:
 
6376
            self.Pdirs = [pjoin(self.me_dir, 'SubProcesses', l.strip()) 
 
6377
                     for l in open(pjoin(self.me_dir,'SubProcesses', 'subproc.mg'))]
 
6378
        else:
 
6379
            self.Pdirs = [l.strip() 
 
6380
                     for l in open(pjoin(self.me_dir,'SubProcesses', 'subproc.mg'))] 
 
6381
          
 
6382
        return self.Pdirs
 
6383
        
 
6384
    def prepare_local_dir(self):
 
6385
        """create the P directory structure in the local directory"""
 
6386
        
 
6387
        if not self.readonly:
 
6388
            os.chdir(self.me_dir)
 
6389
        else:
 
6390
            for line in open(pjoin(self.me_dir,'SubProcesses','subproc.mg')):
 
6391
                os.mkdir(line.strip())
 
6392
            
 
6393
 
6368
6394
    def launch(self, nb_event, seed):
6369
6395
        """ launch the generation for the grid """
6370
6396
 
6371
6397
        # 1) Restore the default data
6372
6398
        logger.info('generate %s events' % nb_event)
6373
6399
        self.set_run_name('GridRun_%s' % seed)
6374
 
        self.update_status('restoring default data', level=None)
6375
 
        misc.call([pjoin(self.me_dir,'bin','internal','restore_data'),
6376
 
                         'default'],
6377
 
            cwd=self.me_dir)
 
6400
        if not self.readonly:
 
6401
            self.update_status('restoring default data', level=None)
 
6402
            misc.call([pjoin(self.me_dir,'bin','internal','restore_data'),
 
6403
                         'default'], cwd=self.me_dir)
6378
6404
 
6379
6405
        # 2) Run the refine for the grid
6380
6406
        self.update_status('Generating Events', level=None)
6385
6411
 
6386
6412
        # 3) Combine the events/pythia/...
6387
6413
        self.exec_cmd('combine_events')
6388
 
        self.exec_cmd('store_events')
6389
 
        self.print_results_in_shell(self.results.current)
6390
 
        self.exec_cmd('decay_events -from_cards', postcmd=False)
 
6414
        if not self.readonly:
 
6415
            self.exec_cmd('store_events')
 
6416
            self.print_results_in_shell(self.results.current)
 
6417
        else:
 
6418
            self.exec_cmd('decay_events -from_cards', postcmd=False)
6391
6419
 
6392
6420
    def refine4grid(self, nb_event):
6393
6421
        """Special refine for gridpack run."""
6395
6423
        
6396
6424
        precision = nb_event
6397
6425
 
 
6426
        self.opts = dict([(key,value[1]) for (key,value) in \
 
6427
                          self._survey_options.items()])
 
6428
        
6398
6429
        # initialize / remove lhapdf mode
6399
6430
        # self.configure_directory() # All this has been done before
6400
6431
        self.cluster_mode = 0 # force single machine
6404
6435
        
6405
6436
        self.update_status('Refine results to %s' % precision, level=None)
6406
6437
        logger.info("Using random number seed offset = %s" % self.random)
6407
 
        
 
6438
 
 
6439
        refine_opt = {'err_goal': nb_event, 'split_channels': False,
 
6440
                      'ngran':self.granularity, 'readonly': self.readonly}   
 
6441
        x_improve = gen_ximprove.gen_ximprove_gridpack(self, refine_opt)
 
6442
        x_improve.launch() # create the ajob for the refinment and run those!
 
6443
        self.gscalefact = x_improve.gscalefact #store jacobian associate to the gridpack 
 
6444
        
 
6445
        
 
6446
        #bindir = pjoin(os.path.relpath(self.dirbin, pjoin(self.me_dir,'SubProcesses')))
 
6447
        #print 'run combine!!!'
 
6448
        #combine_runs.CombineRuns(self.me_dir)
 
6449
        
 
6450
        #update html output
 
6451
        Presults = sum_html.collect_result(self)
 
6452
        cross, error = Presults.xsec, Presults.xerru
 
6453
        self.results.add_detail('cross', cross)
 
6454
        self.results.add_detail('error', error)
 
6455
        
 
6456
        
 
6457
        #self.update_status('finish refine', 'parton', makehtml=False)
 
6458
        #devnull.close()
 
6459
        
 
6460
        
 
6461
        
 
6462
        return
6408
6463
        self.total_jobs = 0
6409
6464
        subproc = [P for P in os.listdir(pjoin(self.me_dir,'SubProcesses')) if 
6410
6465
                   P.startswith('P') and os.path.isdir(pjoin(self.me_dir,'SubProcesses', P))]
6462
6517
        self.update_status('finish refine', 'parton', makehtml=False)
6463
6518
        devnull.close()
6464
6519
 
 
6520
    def do_combine_events(self, line):
 
6521
        """Advanced commands: Launch combine events""" 
 
6522
 
 
6523
        if self.readonly:
 
6524
            outdir = 'Events'
 
6525
            if not os.path.exists(outdir):
 
6526
                os.mkdir(outdir)
 
6527
        else:
 
6528
            outdir = pjoin(self.me_dir, 'Events')
 
6529
        args = self.split_arg(line)
 
6530
        # Check argument's validity
 
6531
        self.check_combine_events(args)
 
6532
        gscalefact = self.gscalefact # {(C.get('name')): jac}
 
6533
        # Define The Banner
 
6534
        tag = self.run_card['run_tag']
 
6535
        # Update the banner with the pythia card
 
6536
        if not self.banner:
 
6537
            self.banner = banner_mod.recover_banner(self.results, 'parton')
 
6538
        self.banner.load_basic(self.me_dir)
 
6539
        # Add cross-section/event information
 
6540
        self.banner.add_generation_info(self.results.current['cross'], self.run_card['nevents'])
 
6541
        if not hasattr(self, 'random_orig'): self.random_orig = 0
 
6542
        self.banner.change_seed(self.random_orig)
 
6543
        
 
6544
        
 
6545
        if not os.path.exists(pjoin(outdir, self.run_name)):
 
6546
                os.mkdir(pjoin(outdir, self.run_name))
 
6547
        self.banner.write(pjoin(outdir, self.run_name, 
 
6548
                                '%s_%s_banner.txt' % (self.run_name, tag)))
 
6549
        
 
6550
        get_wgt = lambda event: event.wgt            
 
6551
        AllEvent = lhe_parser.MultiEventFile()
 
6552
        AllEvent.banner = self.banner
 
6553
        
 
6554
        partials = 0 # if too many file make some partial unweighting
 
6555
        sum_xsec, sum_xerru, sum_axsec = 0,[],0
 
6556
        for Gdir in self.get_Gdir():
 
6557
            #mfactor already taken into accoun in auto_dsig.f
 
6558
            if os.path.exists(pjoin(Gdir, 'events.lhe')):
 
6559
                result = sum_html.OneResult('')
 
6560
                result.read_results(pjoin(Gdir, 'results.dat'))
 
6561
                AllEvent.add(pjoin(Gdir, 'events.lhe'), 
 
6562
                             result.get('xsec')*gscalefact[Gdir],
 
6563
                             result.get('xerru')*gscalefact[Gdir],
 
6564
                             result.get('axsec')*gscalefact[Gdir]
 
6565
                             )
 
6566
 
 
6567
                sum_xsec += result.get('xsec')*gscalefact[Gdir]
 
6568
                sum_xerru.append(result.get('xerru')*gscalefact[Gdir])
 
6569
                sum_axsec += result.get('axsec')*gscalefact[Gdir]
 
6570
                
 
6571
                if len(AllEvent) >= 80: #perform a partial unweighting
 
6572
                    AllEvent.unweight(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % partials),
 
6573
                          get_wgt, log_level=5,  trunc_error=1e-2, event_target=self.nb_event)
 
6574
                    AllEvent = lhe_parser.MultiEventFile()
 
6575
                    AllEvent.banner = self.banner
 
6576
                    AllEvent.add(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % partials),
 
6577
                                 sum_xsec,
 
6578
                                 math.sqrt(sum(x**2 for x in sum_xerru)),
 
6579
                                 sum_axsec) 
 
6580
                    partials +=1
 
6581
        
 
6582
        if not hasattr(self,'proc_characteristic'):
 
6583
            self.proc_characteristic = self.get_characteristics()
 
6584
            
 
6585
        nb_event = AllEvent.unweight(pjoin(outdir, self.run_name, "unweighted_events.lhe.gz"),
 
6586
                          get_wgt, trunc_error=1e-2, event_target=self.nb_event,
 
6587
                          log_level=logging.DEBUG, normalization=self.run_card['event_norm'],
 
6588
                          proc_charac=self.proc_characteristic)
 
6589
        
 
6590
        
 
6591
        if partials:
 
6592
            for i in range(partials):
 
6593
                try:
 
6594
                    os.remove(pjoin(outdir, self.run_name, "partials%s.lhe.gz" % i))
 
6595
                except Exception:
 
6596
                    os.remove(pjoin(outdir, self.run_name, "partials%s.lhe" % i))
 
6597
                   
 
6598
        self.results.add_detail('nb_event', nb_event)
 
6599
    
 
6600
        if self.run_card['bias_module'].lower() not in  ['dummy', 'none']:
 
6601
            self.correct_bias()
 
6602
 
 
6603
    def do_combine_events_v4(self, line):
 
6604
        """Advanced commands: Launch combine events"""    
 
6605
        
 
6606
        args = self.split_arg(line)
 
6607
    
 
6608
        # Check argument's validity
 
6609
        self.check_combine_events(args)
 
6610
 
 
6611
        self.update_status('Combining Events', level='parton')
 
6612
 
 
6613
        try:
 
6614
            os.remove(pjoin(self.me_dir,'SubProcesses', 'combine.log'))
 
6615
        except Exception:
 
6616
            pass
 
6617
        
 
6618
        if not self.readonly:        
 
6619
            run_dir = pjoin(self.me_dir,'SubProcesses')
 
6620
            stdout_file = pjoin(self.me_dir,'SubProcesses', 'combine.log')
 
6621
        else:
 
6622
            run_dir = pjoin('SubProcesses')
 
6623
            stdout_file = pjoin('SubProcesses', 'combine.log')
 
6624
 
 
6625
        cluster.onecore.launch_and_wait('../bin/internal/run_combine', 
 
6626
                                       args=[self.run_name],
 
6627
                                       cwd=run_dir,
 
6628
                                       stdout=stdout_file,
 
6629
                                       required_output=[pjoin(self.me_dir,'SubProcesses', 'combine.log')])
 
6630
            
 
6631
        output = misc.mult_try_open(stdout_file).read()
 
6632
        # Store the number of unweighted events for the results object
 
6633
        pat = re.compile(r'''\s*Unweighting\s*selected\s*(\d+)\s*events''')
 
6634
        try:      
 
6635
            nb_event = pat.search(output).groups()[0]
 
6636
        except AttributeError:
 
6637
            time.sleep(10)
 
6638
            output = misc.mult_try_open(pjoin(self.me_dir,'SubProcesses','combine.log')).read()
 
6639
            try:
 
6640
                nb_event = pat.search(output).groups()[0]
 
6641
            except AttributeError:
 
6642
                logger.warning('Fail to read the number of unweighted events in the combine.log file')
 
6643
                nb_event = 0
 
6644
        self.results.add_detail('nb_event', nb_event)
 
6645
        
 
6646
        # Define The Banner
 
6647
        tag = self.run_card['run_tag']
 
6648
        
 
6649
        # Update the banner with the pythia card
 
6650
        if not self.banner:
 
6651
            self.banner = banner_mod.recover_banner(self.results, 'parton')
 
6652
        self.banner.load_basic(self.me_dir)
 
6653
        # Add cross-section/event information
 
6654
        self.banner.add_generation_info(self.results.current['cross'], nb_event)
 
6655
        if not hasattr(self, 'random_orig'): self.random_orig = 0
 
6656
        self.banner.change_seed(self.random_orig)
 
6657
        if not os.path.exists(pjoin(self.me_dir, 'Events', self.run_name)):
 
6658
            os.mkdir(pjoin(self.me_dir, 'Events', self.run_name))
 
6659
        self.banner.write(pjoin(self.me_dir, 'Events', self.run_name, 
 
6660
                                     '%s_%s_banner.txt' % (self.run_name, tag)))
 
6661
        
 
6662
        
 
6663
        self.banner.add_to_file(pjoin(self.me_dir,'Events', 'events.lhe'),
 
6664
                                out=pjoin(self.me_dir,'Events', self.run_name, 'events.lhe'))
 
6665
        self.banner.add_to_file(pjoin(self.me_dir,'Events', 'unweighted_events.lhe'),        
 
6666
                                out=pjoin(self.me_dir,'Events', self.run_name, 'unweighted_events.lhe'))        
 
6667
 
6465
6668
 
6466
6669
class MadLoopInitializer(object):
6467
6670
    """ A container class for the various methods for initializing MadLoop. It is
6917
7120
            level = int(options.logging)
6918
7121
        else:
6919
7122
            level = eval('logging.' + options.logging)
6920
 
        print os.path.join(root_path, 'internal', 'me5_logging.conf')
6921
7123
        logging.config.fileConfig(os.path.join(root_path, 'internal', 'me5_logging.conf'))
6922
7124
        logging.root.setLevel(level)
6923
7125
        logging.getLogger('madgraph').setLevel(level)