← Back to branch summary

~madteam/mg5amcnlo/series2.0

« back to all changes in this revision

Viewing changes to madgraph/interface/madevent_interface.py

merge lp:~maddevelopers/madgraph5/transfer_on_exit 

Show diffs side-by-side

added added

removed removed

Lines of Context:
madgraph/interface/madevent_interface.py
61
61
    import madgraph.various.gen_crossxhtml as gen_crossxhtml
62
62
    import madgraph.various.sum_html as sum_html
63
63
    import madgraph.various.misc as misc
 
64
    import madgraph.various.combine_runs as combine_runs
64
65
 
65
66
    import models.check_param_card as check_param_card    
66
67
    from madgraph import InvalidCmd, MadGraph5Error
79
80
    import internal.cluster as cluster
80
81
    import internal.check_param_card as check_param_card
81
82
    import internal.sum_html as sum_html
 
83
    import internal.combine_runs as combine_runs
82
84
    MADEVENT = True
83
85
 
84
86
class MadEventError(Exception):
338
340
        logger.info("   timeout VALUE")
339
341
        logger.info("      (default 20) Seconds allowed to answer questions.")
340
342
        logger.info("      Note that pressing tab always stops the timer.")        
341
 
 
 
343
        logger.info("   cluster_temp_path PATH")
 
344
        logger.info("      (default None) Allow to perform the run in PATH directory")
 
345
        logger.info("      This allow to not run on the central disk. This is not used")
 
346
        logger.info("      by condor cluster (since condor has it's own way to prevent it).")
 
347
        
 
348
        
342
349
    def run_options_help(self, data):
343
350
        if data:
344
351
            logger.info('-- local options:')
597
604
                        
598
605
    def check_set(self, args):
599
606
        """ check the validity of the line"""
600
 
 
 
607
        
601
608
        if len(args) < 2:
602
609
            self.help_set()
603
610
            raise self.InvalidCmd('set needs an option and an argument')
1539
1546
    options_madevent = {'automatic_html_opening':True,
1540
1547
                         'run_mode':2,
1541
1548
                         'cluster_queue':'madgraph',
1542
 
                         'nb_core': None}
 
1549
                         'nb_core': None,
 
1550
                         'cluster_temp_path':None}
1543
1551
    
1544
1552
    
1545
1553
    ############################################################################
1643
1651
            else:
1644
1652
                continue
1645
1653
            args.remove(arg)
1646
 
        
 
1654
 
 
1655
        if args and args[0] in ["run_mode", "cluster_mode", "cluster_queue", 
 
1656
                                "cluster_temp_path", "nb_core"]:
 
1657
            return args
 
1658
 
1647
1659
        if self.cluster_mode == 2 and not self.nb_core:
1648
1660
            import multiprocessing
1649
1661
            self.nb_core = multiprocessing.cpu_count()
1650
1662
            
1651
1663
        if self.cluster_mode == 1 and not hasattr(self, 'cluster'):
1652
 
            cluster_name = self.options['cluster_type']
1653
 
            self.cluster = cluster.from_name[cluster_name](self.options['cluster_queue'])
 
1664
            opt = self.options
 
1665
            cluster_name = opt['cluster_type']
 
1666
            self.cluster = cluster.from_name[cluster_name](opt['cluster_queue'],
 
1667
                                                        opt['cluster_temp_path'])
1654
1668
        return args
1655
1669
    
1656
1670
    ############################################################################            
1951
1965
                raise self.InvalidCmd, 'run_mode should be 0, 1 or 2.'
1952
1966
            self.cluster_mode = int(args[1])
1953
1967
            self.options['run_mode'] =  self.cluster_mode
1954
 
        elif args[0] == 'cluster_type':
1955
 
            self.options['cluster_type'] = args[1]
1956
 
            self.cluster = cluster.from_name[args[1]](self.options['cluster_queue'])
 
1968
        elif args[0] in  ['cluster_type', 'cluster_queue', 'cluster_temp_path']:
 
1969
            self.options[args[0]] = args[1]
 
1970
            opt = self.options
 
1971
            self.cluster = cluster.from_name[opt['cluster_type']](\
 
1972
                                 opt['cluster_queue'], opt['cluster_temp_path'])
1957
1973
        elif args[0] == 'nb_core':
1958
1974
            if args[1] == 'None':
1959
1975
                import multiprocessing
2373
2389
            # clean previous run
2374
2390
            for match in glob.glob(pjoin(Pdir, '*ajob*')):
2375
2391
                if os.path.basename(match)[:4] in ['ajob', 'wait', 'run.', 'done']:
2376
 
                    os.remove(pjoin(Pdir, match))
 
2392
                    os.remove(match)
 
2393
            for match in glob.glob(pjoin(Pdir, 'G*')):
 
2394
                if os.path.exists(pjoin(match,'results.dat')):
 
2395
                    os.remove(pjoin(match, 'results.dat'))
2377
2396
            
2378
2397
            #compile gensym
2379
2398
            misc.compile(['gensym'], cwd=Pdir)
2401
2420
            self.total_jobs += len(alljobs)
2402
2421
            for i, job in enumerate(alljobs):
2403
2422
                job = os.path.basename(job)
2404
 
                self.launch_job('./%s' % job, cwd=Pdir, remaining=(len(alljobs)-i-1), 
 
2423
                self.launch_job('%s' % job, cwd=Pdir, remaining=(len(alljobs)-i-1), 
2405
2424
                                                    run_type='survey on %s (%s/%s)' % (subdir,nb_proc+1,len(subproc)))
2406
2425
                if os.path.exists(pjoin(self.me_dir,'error')):
2407
2426
                    self.monitor(html=True)
2453
2472
            # clean previous run
2454
2473
            for match in glob.glob(pjoin(Pdir, '*ajob*')):
2455
2474
                if os.path.basename(match)[:4] in ['ajob', 'wait', 'run.', 'done']:
2456
 
                    os.remove(pjoin(Pdir, match))
 
2475
                    os.remove(match)
2457
2476
            
2458
2477
            proc = misc.Popen([pjoin(bindir, 'gen_ximprove')],
2459
2478
                                    stdout=devnull,
2463
2482
 
2464
2483
            if os.path.exists(pjoin(Pdir, 'ajob1')):
2465
2484
                misc.compile(['madevent'], cwd=Pdir)
2466
 
                #
2467
2485
                alljobs = glob.glob(pjoin(Pdir,'ajob*'))
 
2486
                
 
2487
                #remove associated results.dat (ensure to not mix with all data)
 
2488
                Gre = re.compile("\s*j=(G[\d\.\w]+)")
 
2489
                for job in alljobs:
 
2490
                    Gdirs = Gre.findall(open(job).read())
 
2491
                    for Gdir in Gdirs:
 
2492
                        if os.path.exists(pjoin(Pdir, Gdir, 'results.dat')):
 
2493
                            os.remove(pjoin(Pdir, Gdir,'results.dat'))
 
2494
                
2468
2495
                nb_tot = len(alljobs)            
2469
2496
                self.total_jobs += nb_tot
2470
2497
                for i, job in enumerate(alljobs):
2471
2498
                    job = os.path.basename(job)
2472
 
                    self.launch_job('./%s' % job, cwd=Pdir, remaining=(nb_tot-i-1), 
 
2499
                    self.launch_job('%s' % job, cwd=Pdir, remaining=(nb_tot-i-1), 
2473
2500
                             run_type='Refine number %s on %s (%s/%s)' % 
2474
2501
                             (self.nb_refine, subdir, nb_proc+1, len(subproc)))
2475
2502
        self.monitor(run_type='All job submitted for refine number %s' % self.nb_refine, 
2482
2509
            pass
2483
2510
        
2484
2511
        bindir = pjoin(os.path.relpath(self.dirbin, pjoin(self.me_dir,'SubProcesses')))
2485
 
        misc.call([pjoin(bindir, 'combine_runs')], 
2486
 
                                          cwd=pjoin(self.me_dir,'SubProcesses'),
2487
 
                                          stdout=devnull)
 
2512
 
 
2513
        combine_runs.CombineRuns(self.me_dir)
2488
2514
        
2489
2515
        cross, error = sum_html.make_all_html_results(self)
2490
2516
        self.results.add_detail('cross', cross)
2513
2539
            out = misc.call(['../bin/internal/run_combine'],
2514
2540
                         cwd=pjoin(self.me_dir,'SubProcesses'), 
2515
2541
                         stdout=open(pjoin(self.me_dir,'SubProcesses','combine.log'),'w'))
2516
 
            
2517
 
        output = open(pjoin(self.me_dir,'SubProcesses','combine.log')).read()
 
2542
        
 
2543
        
 
2544
        output = misc.mult_try_open(pjoin(self.me_dir,'SubProcesses','combine.log')).read()
2518
2545
        # Store the number of unweighted events for the results object
2519
2546
        pat = re.compile(r'''\s*Unweighting selected\s*(\d+)\s*events''',re.MULTILINE)
2520
 
        if self.cluster_mode == 1 and not output:
2521
 
            time.sleep(30)
2522
 
            output = open(pjoin(self.me_dir,'SubProcesses','combine.log')).read()
2523
 
            
2524
 
        if output:  
2525
 
            nb_event = pat.search(output).groups()[0]
2526
 
            self.results.add_detail('nb_event', nb_event)
2527
 
        else :
2528
 
            nb_event =0
 
2547
              
 
2548
        nb_event = pat.search(output).groups()[0]
 
2549
        self.results.add_detail('nb_event', nb_event)
 
2550
        
2529
2551
        
2530
2552
        # Define The Banner
2531
2553
        tag = self.run_card['run_tag']
3372
3394
 
3373
3395
 
3374
3396
        elif mode == 1:
3375
 
            self.cluster.submit(exe, stdout=stdout, cwd=cwd)
 
3397
            # For condor cluster, create the input/output files
 
3398
            if 'ajob' in exe: 
 
3399
                input_files = ['madevent','input_app.txt','symfact.dat','iproc.dat',
 
3400
                               pjoin(self.me_dir, 'SubProcesses','randinit')]
 
3401
                output_files = []
 
3402
                
 
3403
                #Find the correct PDF input file
 
3404
                if self.pdffile:
 
3405
                    input_files.append(self.pdffile)
 
3406
                else:
 
3407
                    for line in open(pjoin(self.me_dir,'Source','PDF','pdf_list.txt')):
 
3408
                        data = line.split()
 
3409
                        if len(data) < 4:
 
3410
                            continue
 
3411
                        if data[0].lower() == self.run_card['pdlabel'].lower():
 
3412
                            self.pdffile = pjoin(self.me_dir, 'lib', 'Pdfdata', data[2])
 
3413
                            input_files.append(self.pdffile) 
 
3414
                            break
 
3415
                    else:
 
3416
                        # possible when using lhapdf
 
3417
                        self.pdffile = pjoin(self.me_dir, 'lib', 'PDFsets')
 
3418
                        input_files.append(self.pdffile) 
 
3419
                        
 
3420
                
 
3421
                #Find the correct ajob
 
3422
                Gre = re.compile("\s*j=(G[\d\.\w]+)")
 
3423
                Ire = re
 
3424
                try : 
 
3425
                    fsock = open(exe)
 
3426
                except:
 
3427
                    fsock = open(pjoin(cwd,exe))
 
3428
                text = fsock.read()
 
3429
                output_files = Gre.findall(text)
 
3430
                if not output_files:
 
3431
                    Ire = re.compile("for i in ([\d\s]*) ; do")
 
3432
                    data = Ire.findall(text)
 
3433
                    data = ' '.join(data).split()
 
3434
                    for nb in data:
 
3435
                        output_files.append('G%s' % nb)
 
3436
                else:
 
3437
                    for G in output_files:
 
3438
                        if os.path.isdir(pjoin(cwd,G)):
 
3439
                            input_files.append(G)
 
3440
                
 
3441
                #submitting
 
3442
                self.cluster.submit2(exe, stdout=stdout, cwd=cwd, 
 
3443
                             input_files=input_files, output_files=output_files)
 
3444
            
 
3445
            else:
 
3446
                self.cluster.submit(exe, stdout=stdout, cwd=cwd)
3376
3447
 
3377
3448
        elif mode == 2:
3378
3449
            import thread
3490
3561
        time_mod = max([os.path.getctime(pjoin(self.me_dir,'Cards','run_card.dat')),
3491
3562
                        os.path.getctime(pjoin(self.me_dir,'Cards','param_card.dat'))])
3492
3563
        
3493
 
        if self.configured > time_mod:
 
3564
        if self.configured > time_mod and hasattr(self, 'random'):
3494
3565
            return
3495
3566
        else:
3496
3567
            self.configured = time.time()
3518
3589
            os.environ['lhapdf'] = 'True'
3519
3590
        elif 'lhapdf' in os.environ.keys():
3520
3591
            del os.environ['lhapdf']
 
3592
        self.pdffile = None
3521
3593
            
3522
3594
        # set random number
3523
3595
        if self.run_card['iseed'] != '0':
3546
3618
        
3547
3619
        # Compile
3548
3620
        for name in ['../bin/internal/gen_ximprove', 'all', 
3549
 
                     '../bin/internal/combine_events', 
3550
 
                     '../bin/internal/combine_runs']:
 
3621
                     '../bin/internal/combine_events']:
3551
3622
            misc.compile(arg=[name], cwd=os.path.join(self.me_dir, 'Source'))
3552
3623
        
3553
3624
        
4510
4581
                self.total_jobs += nb_tot
4511
4582
                for i, job in enumerate(alljobs):
4512
4583
                    job = os.path.basename(job)
4513
 
                    self.launch_job('./%s' % job, cwd=Pdir, remaining=(nb_tot-i-1), 
 
4584
                    self.launch_job('%s' % job, cwd=Pdir, remaining=(nb_tot-i-1), 
4514
4585
                             run_type='Refine number %s on %s (%s/%s)' %
4515
4586
                             (self.nb_refine, subdir, nb_proc+1, len(subproc)))
4516
4587
                    if os.path.exists(pjoin(self.me_dir,'error')):