← Back to branch summary

~ubuntu-branches/ubuntu/vivid/emscripten/vivid

« back to all changes in this revision

Viewing changes to third_party/ply/test/testlex.py

  • Committer: Package Import Robot
  • Author(s): Sylvestre Ledru
  • Date: 2013-05-02 13:11:51 UTC
  • Revision ID: package-import@ubuntu.com-20130502131151-q8dvteqr1ef2x7xz
Tags: upstream-1.4.1~20130504~adb56cb
Import upstream version 1.4.1~20130504~adb56cb

Show diffs side-by-side

added added

removed removed

Lines of Context:
third_party/ply/test/testlex.py
 
1
# testlex.py
 
2
 
 
3
import unittest
 
4
try:
 
5
    import StringIO
 
6
except ImportError:
 
7
    import io as StringIO
 
8
 
 
9
import sys
 
10
import os
 
11
import imp
 
12
import warnings
 
13
 
 
14
sys.path.insert(0,"..")
 
15
sys.tracebacklimit = 0
 
16
 
 
17
import ply.lex
 
18
 
 
19
def make_pymodule_path(filename):
 
20
    path = os.path.dirname(filename)
 
21
    file = os.path.basename(filename)
 
22
    mod, ext = os.path.splitext(file)
 
23
 
 
24
    if sys.hexversion >= 0x3020000:
 
25
        modname = mod+"."+imp.get_tag()+ext
 
26
        fullpath = os.path.join(path,'__pycache__',modname)
 
27
    else:
 
28
        fullpath = filename
 
29
    return fullpath
 
30
 
 
31
def pymodule_out_exists(filename):
 
32
    return os.path.exists(make_pymodule_path(filename))
 
33
 
 
34
def pymodule_out_remove(filename):
 
35
    os.remove(make_pymodule_path(filename))
 
36
 
 
37
def check_expected(result,expected):
 
38
    if sys.version_info[0] >= 3:
 
39
        if isinstance(result,str):
 
40
            result = result.encode('ascii')
 
41
        if isinstance(expected,str):
 
42
            expected = expected.encode('ascii')
 
43
    resultlines = result.splitlines()
 
44
    expectedlines = expected.splitlines()
 
45
 
 
46
 
 
47
    if len(resultlines) != len(expectedlines):
 
48
        return False
 
49
 
 
50
    for rline,eline in zip(resultlines,expectedlines):
 
51
        if not rline.endswith(eline):
 
52
            return False
 
53
    return True
 
54
 
 
55
def run_import(module):
 
56
    code = "import "+module
 
57
    exec(code)
 
58
    del sys.modules[module]
 
59
    
 
60
# Tests related to errors and warnings when building lexers
 
61
class LexErrorWarningTests(unittest.TestCase):
 
62
    def setUp(self):
 
63
        sys.stderr = StringIO.StringIO()
 
64
        sys.stdout = StringIO.StringIO()
 
65
        if sys.hexversion >= 0x3020000:
 
66
            warnings.filterwarnings('ignore',category=ResourceWarning)
 
67
 
 
68
    def tearDown(self):
 
69
        sys.stderr = sys.__stderr__
 
70
        sys.stdout = sys.__stdout__
 
71
    def test_lex_doc1(self):
 
72
        self.assertRaises(SyntaxError,run_import,"lex_doc1")
 
73
        result = sys.stderr.getvalue()
 
74
        self.assert_(check_expected(result,
 
75
                              "lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\n"))
 
76
    def test_lex_dup1(self):
 
77
        self.assertRaises(SyntaxError,run_import,"lex_dup1")
 
78
        result = sys.stderr.getvalue()
 
79
        self.assert_(check_expected(result,
 
80
                                    "lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))        
 
81
        
 
82
    def test_lex_dup2(self):
 
83
        self.assertRaises(SyntaxError,run_import,"lex_dup2")
 
84
        result = sys.stderr.getvalue()
 
85
        self.assert_(check_expected(result,
 
86
                                    "lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
 
87
            
 
88
    def test_lex_dup3(self):
 
89
        self.assertRaises(SyntaxError,run_import,"lex_dup3")
 
90
        result = sys.stderr.getvalue()
 
91
        self.assert_(check_expected(result,
 
92
                                    "lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
 
93
 
 
94
    def test_lex_empty(self):
 
95
        self.assertRaises(SyntaxError,run_import,"lex_empty")
 
96
        result = sys.stderr.getvalue()
 
97
        self.assert_(check_expected(result,
 
98
                                    "No rules of the form t_rulename are defined\n"
 
99
                                    "No rules defined for state 'INITIAL'\n"))
 
100
 
 
101
    def test_lex_error1(self):
 
102
        run_import("lex_error1")
 
103
        result = sys.stderr.getvalue()
 
104
        self.assert_(check_expected(result,
 
105
                                    "No t_error rule is defined\n"))
 
106
 
 
107
    def test_lex_error2(self):
 
108
        self.assertRaises(SyntaxError,run_import,"lex_error2")
 
109
        result = sys.stderr.getvalue()
 
110
        self.assert_(check_expected(result,
 
111
                                    "Rule 't_error' must be defined as a function\n")
 
112
                     )
 
113
 
 
114
    def test_lex_error3(self):
 
115
        self.assertRaises(SyntaxError,run_import,"lex_error3")
 
116
        result = sys.stderr.getvalue()
 
117
        self.assert_(check_expected(result,
 
118
                                    "lex_error3.py:20: Rule 't_error' requires an argument\n"))
 
119
 
 
120
    def test_lex_error4(self):
 
121
        self.assertRaises(SyntaxError,run_import,"lex_error4")
 
122
        result = sys.stderr.getvalue()
 
123
        self.assert_(check_expected(result,
 
124
                                    "lex_error4.py:20: Rule 't_error' has too many arguments\n"))
 
125
 
 
126
    def test_lex_ignore(self):
 
127
        self.assertRaises(SyntaxError,run_import,"lex_ignore")
 
128
        result = sys.stderr.getvalue()
 
129
        self.assert_(check_expected(result,
 
130
                                    "lex_ignore.py:20: Rule 't_ignore' must be defined as a string\n"))
 
131
 
 
132
    def test_lex_ignore2(self):
 
133
        run_import("lex_ignore2")
 
134
        result = sys.stderr.getvalue()
 
135
        self.assert_(check_expected(result,
 
136
                                    "t_ignore contains a literal backslash '\\'\n"))
 
137
 
 
138
 
 
139
    def test_lex_re1(self):
 
140
        self.assertRaises(SyntaxError,run_import,"lex_re1")
 
141
        result = sys.stderr.getvalue()
 
142
        self.assert_(check_expected(result,
 
143
                                    "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n"))
 
144
 
 
145
    def test_lex_re2(self):
 
146
        self.assertRaises(SyntaxError,run_import,"lex_re2")
 
147
        result = sys.stderr.getvalue()
 
148
        self.assert_(check_expected(result,
 
149
                                    "Regular expression for rule 't_PLUS' matches empty string\n"))
 
150
 
 
151
    def test_lex_re3(self):
 
152
        self.assertRaises(SyntaxError,run_import,"lex_re3")
 
153
        result = sys.stderr.getvalue()
 
154
        self.assert_(check_expected(result,
 
155
                                    "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
 
156
                                    "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n"))
 
157
 
 
158
    def test_lex_rule1(self):
 
159
        self.assertRaises(SyntaxError,run_import,"lex_rule1")
 
160
        result = sys.stderr.getvalue()
 
161
        self.assert_(check_expected(result,
 
162
                                    "t_NUMBER not defined as a function or string\n"))
 
163
 
 
164
    def test_lex_rule2(self):
 
165
        self.assertRaises(SyntaxError,run_import,"lex_rule2")
 
166
        result = sys.stderr.getvalue()
 
167
        self.assert_(check_expected(result,
 
168
                                    "lex_rule2.py:18: Rule 't_NUMBER' requires an argument\n"))
 
169
 
 
170
    def test_lex_rule3(self):
 
171
        self.assertRaises(SyntaxError,run_import,"lex_rule3")
 
172
        result = sys.stderr.getvalue()
 
173
        self.assert_(check_expected(result,
 
174
                                    "lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\n"))
 
175
 
 
176
 
 
177
    def test_lex_state1(self):
 
178
        self.assertRaises(SyntaxError,run_import,"lex_state1")
 
179
        result = sys.stderr.getvalue()
 
180
        self.assert_(check_expected(result,
 
181
                                   "states must be defined as a tuple or list\n"))
 
182
 
 
183
    def test_lex_state2(self):
 
184
        self.assertRaises(SyntaxError,run_import,"lex_state2")
 
185
        result = sys.stderr.getvalue()
 
186
        self.assert_(check_expected(result,
 
187
                                    "Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\n"
 
188
                                    "Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\n"))
 
189
 
 
190
    def test_lex_state3(self):
 
191
        self.assertRaises(SyntaxError,run_import,"lex_state3")
 
192
        result = sys.stderr.getvalue()
 
193
        self.assert_(check_expected(result,
 
194
                                    "State name 1 must be a string\n"
 
195
                                    "No rules defined for state 'example'\n"))
 
196
 
 
197
    def test_lex_state4(self):
 
198
        self.assertRaises(SyntaxError,run_import,"lex_state4")
 
199
        result = sys.stderr.getvalue()
 
200
        self.assert_(check_expected(result,
 
201
                                    "State type for state comment must be 'inclusive' or 'exclusive'\n"))
 
202
 
 
203
 
 
204
    def test_lex_state5(self):
 
205
        self.assertRaises(SyntaxError,run_import,"lex_state5")
 
206
        result = sys.stderr.getvalue()
 
207
        self.assert_(check_expected(result,
 
208
                                    "State 'comment' already defined\n"))
 
209
 
 
210
    def test_lex_state_noerror(self):
 
211
        run_import("lex_state_noerror")
 
212
        result = sys.stderr.getvalue()
 
213
        self.assert_(check_expected(result,
 
214
                                    "No error rule is defined for exclusive state 'comment'\n"))    
 
215
 
 
216
    def test_lex_state_norule(self):
 
217
        self.assertRaises(SyntaxError,run_import,"lex_state_norule")
 
218
        result = sys.stderr.getvalue()
 
219
        self.assert_(check_expected(result,
 
220
                                    "No rules defined for state 'example'\n"))
 
221
 
 
222
    def test_lex_token1(self):
 
223
        self.assertRaises(SyntaxError,run_import,"lex_token1")
 
224
        result = sys.stderr.getvalue()
 
225
        self.assert_(check_expected(result,
 
226
                                    "No token list is defined\n"
 
227
                                    "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
 
228
                                    "Rule 't_PLUS' defined for an unspecified token PLUS\n"
 
229
                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"
 
230
))
 
231
 
 
232
    def test_lex_token2(self):
 
233
        self.assertRaises(SyntaxError,run_import,"lex_token2")
 
234
        result = sys.stderr.getvalue()
 
235
        self.assert_(check_expected(result,
 
236
                                    "tokens must be a list or tuple\n"
 
237
                                    "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
 
238
                                    "Rule 't_PLUS' defined for an unspecified token PLUS\n"
 
239
                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"
 
240
))
 
241
    
 
242
    def test_lex_token3(self):
 
243
        self.assertRaises(SyntaxError,run_import,"lex_token3")
 
244
        result = sys.stderr.getvalue()
 
245
        self.assert_(check_expected(result,
 
246
                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"))
 
247
 
 
248
 
 
249
    def test_lex_token4(self):
 
250
        self.assertRaises(SyntaxError,run_import,"lex_token4")
 
251
        result = sys.stderr.getvalue()
 
252
        self.assert_(check_expected(result,
 
253
                                    "Bad token name '-'\n"))
 
254
 
 
255
 
 
256
    def test_lex_token5(self):
 
257
        try:
 
258
            run_import("lex_token5")
 
259
        except ply.lex.LexError:
 
260
            e = sys.exc_info()[1]
 
261
        self.assert_(check_expected(str(e),"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'"))
 
262
 
 
263
    def test_lex_token_dup(self):
 
264
        run_import("lex_token_dup")
 
265
        result = sys.stderr.getvalue()
 
266
        self.assert_(check_expected(result,
 
267
                                    "Token 'MINUS' multiply defined\n"))   
 
268
 
 
269
        
 
270
    def test_lex_literal1(self):
 
271
        self.assertRaises(SyntaxError,run_import,"lex_literal1")
 
272
        result = sys.stderr.getvalue()
 
273
        self.assert_(check_expected(result,
 
274
                                    "Invalid literal '**'. Must be a single character\n"))
 
275
 
 
276
    def test_lex_literal2(self):
 
277
        self.assertRaises(SyntaxError,run_import,"lex_literal2")
 
278
        result = sys.stderr.getvalue()
 
279
        self.assert_(check_expected(result,
 
280
                                    "Invalid literals specification. literals must be a sequence of characters\n"))
 
281
 
 
282
import os
 
283
import subprocess
 
284
import shutil
 
285
 
 
286
# Tests related to various build options associated with lexers
 
287
class LexBuildOptionTests(unittest.TestCase):
 
288
    def setUp(self):
 
289
        sys.stderr = StringIO.StringIO()
 
290
        sys.stdout = StringIO.StringIO()
 
291
    def tearDown(self):
 
292
        sys.stderr = sys.__stderr__
 
293
        sys.stdout = sys.__stdout__
 
294
        try:
 
295
            shutil.rmtree("lexdir")
 
296
        except OSError:
 
297
            pass
 
298
 
 
299
    def test_lex_module(self):
 
300
        run_import("lex_module")
 
301
        result = sys.stdout.getvalue()
 
302
        self.assert_(check_expected(result,
 
303
                                    "(NUMBER,3,1,0)\n"
 
304
                                    "(PLUS,'+',1,1)\n"
 
305
                                    "(NUMBER,4,1,2)\n"))
 
306
        
 
307
    def test_lex_object(self):
 
308
        run_import("lex_object")
 
309
        result = sys.stdout.getvalue()
 
310
        self.assert_(check_expected(result,
 
311
                                    "(NUMBER,3,1,0)\n"
 
312
                                    "(PLUS,'+',1,1)\n"
 
313
                                    "(NUMBER,4,1,2)\n"))
 
314
 
 
315
    def test_lex_closure(self):
 
316
        run_import("lex_closure")
 
317
        result = sys.stdout.getvalue()
 
318
        self.assert_(check_expected(result,
 
319
                                    "(NUMBER,3,1,0)\n"
 
320
                                    "(PLUS,'+',1,1)\n"
 
321
                                    "(NUMBER,4,1,2)\n"))
 
322
    def test_lex_optimize(self):
 
323
        try:
 
324
            os.remove("lextab.py")
 
325
        except OSError:
 
326
            pass
 
327
        try:
 
328
            os.remove("lextab.pyc")
 
329
        except OSError:
 
330
            pass
 
331
        try:
 
332
            os.remove("lextab.pyo")
 
333
        except OSError:
 
334
            pass
 
335
        run_import("lex_optimize")
 
336
 
 
337
        result = sys.stdout.getvalue()
 
338
        self.assert_(check_expected(result,
 
339
                                    "(NUMBER,3,1,0)\n"
 
340
                                    "(PLUS,'+',1,1)\n"
 
341
                                    "(NUMBER,4,1,2)\n"))
 
342
        self.assert_(os.path.exists("lextab.py"))
 
343
 
 
344
 
 
345
        p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'],
 
346
                             stdout=subprocess.PIPE)
 
347
        result = p.stdout.read()
 
348
 
 
349
        self.assert_(check_expected(result,
 
350
                                    "(NUMBER,3,1,0)\n"
 
351
                                    "(PLUS,'+',1,1)\n"
 
352
                                    "(NUMBER,4,1,2)\n"))
 
353
        self.assert_(pymodule_out_exists("lextab.pyo"))
 
354
 
 
355
        pymodule_out_remove("lextab.pyo")
 
356
        p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'],
 
357
                             stdout=subprocess.PIPE)
 
358
        result = p.stdout.read()
 
359
        self.assert_(check_expected(result,
 
360
                                    "(NUMBER,3,1,0)\n"
 
361
                                    "(PLUS,'+',1,1)\n"
 
362
                                    "(NUMBER,4,1,2)\n"))
 
363
        self.assert_(pymodule_out_exists("lextab.pyo"))
 
364
        try:
 
365
            os.remove("lextab.py")
 
366
        except OSError:
 
367
            pass
 
368
        try:
 
369
            pymodule_out_remove("lextab.pyc")
 
370
        except OSError:
 
371
            pass
 
372
        try:
 
373
            pymodule_out_remove("lextab.pyo")
 
374
        except OSError:
 
375
            pass
 
376
 
 
377
    def test_lex_optimize2(self):
 
378
        try:
 
379
            os.remove("opt2tab.py")
 
380
        except OSError:
 
381
            pass
 
382
        try:
 
383
            os.remove("opt2tab.pyc")
 
384
        except OSError:
 
385
            pass
 
386
        try:
 
387
            os.remove("opt2tab.pyo")
 
388
        except OSError:
 
389
            pass
 
390
        run_import("lex_optimize2")
 
391
        result = sys.stdout.getvalue()
 
392
        self.assert_(check_expected(result,
 
393
                                    "(NUMBER,3,1,0)\n"
 
394
                                    "(PLUS,'+',1,1)\n"
 
395
                                    "(NUMBER,4,1,2)\n"))
 
396
        self.assert_(os.path.exists("opt2tab.py"))
 
397
 
 
398
        p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'],
 
399
                             stdout=subprocess.PIPE)
 
400
        result = p.stdout.read()
 
401
        self.assert_(check_expected(result,
 
402
                                    "(NUMBER,3,1,0)\n"
 
403
                                    "(PLUS,'+',1,1)\n"
 
404
                                    "(NUMBER,4,1,2)\n"))
 
405
        self.assert_(pymodule_out_exists("opt2tab.pyo"))
 
406
        pymodule_out_remove("opt2tab.pyo")
 
407
        p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'],
 
408
                             stdout=subprocess.PIPE)
 
409
        result = p.stdout.read()
 
410
        self.assert_(check_expected(result,
 
411
                                    "(NUMBER,3,1,0)\n"
 
412
                                    "(PLUS,'+',1,1)\n"
 
413
                                    "(NUMBER,4,1,2)\n"))
 
414
        self.assert_(pymodule_out_exists("opt2tab.pyo"))
 
415
        try:
 
416
            os.remove("opt2tab.py")
 
417
        except OSError:
 
418
            pass
 
419
        try:
 
420
            pymodule_out_remove("opt2tab.pyc")
 
421
        except OSError:
 
422
            pass
 
423
        try:
 
424
            pymodule_out_remove("opt2tab.pyo")
 
425
        except OSError:
 
426
            pass
 
427
 
 
428
    def test_lex_optimize3(self):
 
429
        try:
 
430
            shutil.rmtree("lexdir")
 
431
        except OSError:
 
432
            pass
 
433
        
 
434
        os.mkdir("lexdir")
 
435
        os.mkdir("lexdir/sub")
 
436
        open("lexdir/__init__.py","w").write("")
 
437
        open("lexdir/sub/__init__.py","w").write("")
 
438
        run_import("lex_optimize3")
 
439
        result = sys.stdout.getvalue()
 
440
        self.assert_(check_expected(result,
 
441
                                    "(NUMBER,3,1,0)\n"
 
442
                                    "(PLUS,'+',1,1)\n"
 
443
                                    "(NUMBER,4,1,2)\n"))
 
444
        self.assert_(os.path.exists("lexdir/sub/calctab.py"))
 
445
 
 
446
        p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'],
 
447
                             stdout=subprocess.PIPE)
 
448
        result = p.stdout.read()
 
449
        self.assert_(check_expected(result,
 
450
                                    "(NUMBER,3,1,0)\n"
 
451
                                    "(PLUS,'+',1,1)\n"
 
452
                                    "(NUMBER,4,1,2)\n"))
 
453
        self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo"))
 
454
        pymodule_out_remove("lexdir/sub/calctab.pyo")
 
455
        p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'],
 
456
                             stdout=subprocess.PIPE)
 
457
        result = p.stdout.read()
 
458
        self.assert_(check_expected(result,
 
459
                                    "(NUMBER,3,1,0)\n"
 
460
                                    "(PLUS,'+',1,1)\n"
 
461
                                    "(NUMBER,4,1,2)\n"))
 
462
        self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo"))
 
463
        try:
 
464
            shutil.rmtree("lexdir")
 
465
        except OSError:
 
466
            pass
 
467
 
 
468
    def test_lex_opt_alias(self):
 
469
        try:
 
470
            os.remove("aliastab.py")
 
471
        except OSError:
 
472
            pass
 
473
        try:
 
474
            os.remove("aliastab.pyc")
 
475
        except OSError:
 
476
            pass
 
477
        try:
 
478
            os.remove("aliastab.pyo")
 
479
        except OSError:
 
480
            pass
 
481
        run_import("lex_opt_alias")
 
482
        result = sys.stdout.getvalue()
 
483
        self.assert_(check_expected(result,
 
484
                                    "(NUMBER,3,1,0)\n"
 
485
                                    "(+,'+',1,1)\n"
 
486
                                    "(NUMBER,4,1,2)\n"))
 
487
        self.assert_(os.path.exists("aliastab.py"))
 
488
 
 
489
        p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'],
 
490
                             stdout=subprocess.PIPE)
 
491
        result = p.stdout.read()
 
492
        self.assert_(check_expected(result,
 
493
                                    "(NUMBER,3,1,0)\n"
 
494
                                    "(+,'+',1,1)\n"
 
495
                                    "(NUMBER,4,1,2)\n"))
 
496
        self.assert_(pymodule_out_exists("aliastab.pyo"))
 
497
        pymodule_out_remove("aliastab.pyo")
 
498
        p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'],
 
499
                             stdout=subprocess.PIPE)
 
500
        result = p.stdout.read()
 
501
        self.assert_(check_expected(result,
 
502
                                    "(NUMBER,3,1,0)\n"
 
503
                                    "(+,'+',1,1)\n"
 
504
                                    "(NUMBER,4,1,2)\n"))
 
505
        self.assert_(pymodule_out_exists("aliastab.pyo"))
 
506
        try:
 
507
            os.remove("aliastab.py")
 
508
        except OSError:
 
509
            pass
 
510
        try:
 
511
            pymodule_out_remove("aliastab.pyc")
 
512
        except OSError:
 
513
            pass
 
514
        try:
 
515
            pymodule_out_remove("aliastab.pyo")
 
516
        except OSError:
 
517
            pass
 
518
 
 
519
    def test_lex_many_tokens(self):
 
520
        try:
 
521
            os.remove("manytab.py")
 
522
        except OSError:
 
523
            pass
 
524
        try:
 
525
            os.remove("manytab.pyc")
 
526
        except OSError:
 
527
            pass
 
528
        try:
 
529
            os.remove("manytab.pyo")
 
530
        except OSError:
 
531
            pass
 
532
        run_import("lex_many_tokens")
 
533
        result = sys.stdout.getvalue()
 
534
        self.assert_(check_expected(result,
 
535
                                    "(TOK34,'TOK34:',1,0)\n"
 
536
                                    "(TOK143,'TOK143:',1,7)\n"
 
537
                                    "(TOK269,'TOK269:',1,15)\n"
 
538
                                    "(TOK372,'TOK372:',1,23)\n"
 
539
                                    "(TOK452,'TOK452:',1,31)\n"
 
540
                                    "(TOK561,'TOK561:',1,39)\n"
 
541
                                    "(TOK999,'TOK999:',1,47)\n"
 
542
                                    ))
 
543
 
 
544
        self.assert_(os.path.exists("manytab.py"))
 
545
 
 
546
        p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],
 
547
                             stdout=subprocess.PIPE)
 
548
        result = p.stdout.read()
 
549
        self.assert_(check_expected(result,
 
550
                                    "(TOK34,'TOK34:',1,0)\n"
 
551
                                    "(TOK143,'TOK143:',1,7)\n"
 
552
                                    "(TOK269,'TOK269:',1,15)\n"
 
553
                                    "(TOK372,'TOK372:',1,23)\n"
 
554
                                    "(TOK452,'TOK452:',1,31)\n"
 
555
                                    "(TOK561,'TOK561:',1,39)\n"
 
556
                                    "(TOK999,'TOK999:',1,47)\n"
 
557
                                    ))
 
558
 
 
559
        self.assert_(pymodule_out_exists("manytab.pyo"))
 
560
        pymodule_out_remove("manytab.pyo")
 
561
        try:
 
562
            os.remove("manytab.py")
 
563
        except OSError:
 
564
            pass
 
565
        try:
 
566
            os.remove("manytab.pyc")
 
567
        except OSError:
 
568
            pass
 
569
        try:
 
570
            os.remove("manytab.pyo")
 
571
        except OSError:
 
572
            pass
 
573
        
 
574
# Tests related to run-time behavior of lexers
 
575
class LexRunTests(unittest.TestCase):
 
576
    def setUp(self):
 
577
        sys.stderr = StringIO.StringIO()
 
578
        sys.stdout = StringIO.StringIO()
 
579
    def tearDown(self):
 
580
        sys.stderr = sys.__stderr__
 
581
        sys.stdout = sys.__stdout__
 
582
 
 
583
    def test_lex_hedit(self):
 
584
        run_import("lex_hedit")
 
585
        result = sys.stdout.getvalue()
 
586
        self.assert_(check_expected(result,
 
587
                                    "(H_EDIT_DESCRIPTOR,'abc',1,0)\n"
 
588
                                    "(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\n"
 
589
                                    "(H_EDIT_DESCRIPTOR,'xy',1,20)\n"))
 
590
       
 
591
    def test_lex_state_try(self):
 
592
        run_import("lex_state_try")
 
593
        result = sys.stdout.getvalue()
 
594
        self.assert_(check_expected(result,
 
595
                                    "(NUMBER,'3',1,0)\n"
 
596
                                    "(PLUS,'+',1,2)\n"
 
597
                                    "(NUMBER,'4',1,4)\n"
 
598
                                    "Entering comment state\n"
 
599
                                    "comment body LexToken(body_part,'This is a comment */',1,9)\n"
 
600
                                    "(PLUS,'+',1,30)\n"
 
601
                                    "(NUMBER,'10',1,32)\n"
 
602
                                    ))    
 
603
 
 
604
 
 
605
 
 
606
unittest.main()