4
%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
6
%% The contents of this file are subject to the Erlang Public License,
7
%% Version 1.1, (the "License"); you may not use this file except in
8
%% compliance with the License. You should have received a copy of the
9
%% Erlang Public License along with this software. If not, it can be
10
%% retrieved online at http://www.erlang.org/.
12
%% Software distributed under the License is distributed on an "AS IS"
13
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
14
%% the License for the specific language governing rights and limitations
21
%-define(debug, true).
23
-include_lib("stdlib/include/erl_compile.hrl").
24
-include_lib("kernel/include/file.hrl").
27
-define(line, put(line, ?LINE), ).
28
-define(config(X,Y), foo).
29
-define(datadir, "leex_SUITE_data").
30
-define(privdir, "leex_SUITE_priv").
31
-define(t, test_server).
33
-include_lib("test_server/include/test_server.hrl").
34
-define(datadir, ?config(data_dir, Config)).
35
-define(privdir, ?config(priv_dir, Config)).
38
-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
39
init_per_group/2,end_per_group/2,
40
init_per_testcase/2, end_per_testcase/2]).
43
file/1, compile/1, syntax/1,
45
pt/1, man/1, ex/1, ex2/1, not_yet/1]).
47
% Default timetrap timeout (set in init_per_testcase).
48
-define(default_timeout, ?t:minutes(1)).
50
init_per_testcase(_Case, Config) ->
51
?line Dog = ?t:timetrap(?default_timeout),
52
[{watchdog, Dog} | Config].
54
end_per_testcase(_Case, Config) ->
55
Dog = ?config(watchdog, Config),
56
test_server:timetrap_cancel(Dog),
59
suite() -> [{ct_hooks,[ts_install_cth]}].
62
[{group, checks}, {group, examples}].
65
[{checks, [], [file, compile, syntax]},
66
{examples, [], [pt, man, ex, ex2, not_yet]}].
68
init_per_suite(Config) ->
71
end_per_suite(_Config) ->
74
init_per_group(_GroupName, Config) ->
77
end_per_group(_GroupName, Config) ->
83
"Bad files and options.";
85
file(Config) when is_list(Config) ->
87
Ret = [return, {report, false}],
88
?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} =
89
leex:file("not_a_file", Ret),
90
?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} =
91
leex:file("not_a_file", [{return,true}]),
92
?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} =
93
leex:file("not_a_file", [{report,false},return_errors]),
94
?line error = leex:file("not_a_file"),
95
?line error = leex:file("not_a_file", [{return,false},report]),
96
?line error = leex:file("not_a_file", [return_warnings,{report,false}]),
98
Filename = filename:join(Dir, "file.xrl"),
99
file:delete(Filename),
101
?line {'EXIT', {badarg, _}} = (catch leex:file({foo})),
102
?line {'EXIT', {badarg, _}} =
103
(catch leex:file(Filename, {parserfile,{foo}})),
104
?line {'EXIT', {badarg, _}} =
105
(catch leex:file(Filename, {includefile,{foo}})),
107
?line {'EXIT', {badarg, _}} = (catch leex:file(Filename, no_option)),
108
?line {'EXIT', {badarg, _}} =
109
(catch leex:file(Filename, [return | report])),
110
?line {'EXIT', {badarg, _}} =
111
(catch leex:file(Filename, {return,foo})),
112
?line {'EXIT', {badarg, _}} =
113
(catch leex:file(Filename, includefile)),
115
Mini = <<"Definitions.\n"
118
"{L}+ : {token,{word,TokenLine,TokenChars}}.\n"
120
?line ok = file:write_file(Filename, Mini),
121
?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} =
122
leex:file(Filename, [{scannerfile,"//"} | Ret]),
123
?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} =
124
leex:file(Filename, [{includefile,"//"} | Ret]),
125
?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} =
126
leex:file(Filename, [{includefile,"/ /"} | Ret]),
128
LeexPre = filename:join(Dir, "leexinc.hrl"),
129
?line ok = file:write_file(LeexPre, <<"syntax error.\n">>),
130
PreErrors = run_test(Config, Mini, LeexPre),
132
[{1,_,["syntax error before: ","error"]},
133
{3,_,undefined_module}],
135
extract(LeexPre, PreErrors),
136
file:delete(LeexPre),
138
Ret2 = [return, report_errors, report_warnings, verbose],
139
Scannerfile = filename:join(Dir, "file.erl"),
140
?line ok = file:write_file(Scannerfile, <<"nothing">>),
141
?line unwritable(Scannerfile),
142
?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} =
143
leex:file(Filename, Ret2),
144
?line writable(Scannerfile),
145
file:delete(Scannerfile),
147
Dotfile = filename:join(Dir, "file.dot"),
148
?line ok = file:write_file(Dotfile, <<"nothing">>),
149
?line unwritable(Dotfile),
150
?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} =
151
leex:file(Filename, [dfa_graph | Ret2]),
152
?line writable(Dotfile),
153
file:delete(Dotfile),
155
file:delete(Filename),
159
"Check of compile/3.";
160
compile(suite) -> [];
161
compile(Config) when is_list(Config) ->
163
Filename = filename:join(Dir, "file.xrl"),
164
Scannerfile = filename:join(Dir, "file.erl"),
165
Mini = <<"Definitions.\n"
168
"{L}+ : {token,{word,TokenLine,TokenChars}}.\n"
170
?line ok = file:write_file(Filename, Mini),
171
?line error = leex:compile(Filename, "//", #options{}),
172
?line ok = leex:compile(Filename, Scannerfile, #options{}),
173
file:delete(Scannerfile),
174
file:delete(Filename),
180
syntax(Config) when is_list(Config) ->
182
Filename = filename:join(Dir, "file.xrl"),
183
Ret = [return, {report, true}],
184
?line ok = file:write_file(Filename,
189
"{L}+ : {token,{word,TokenLine,TokenChars}}.\n
191
?line {error,[{_,[{7,leex,missing_code}]}],[]} = leex:file(Filename, Ret),
192
?line ok = file:write_file(Filename,
197
?line {error,[{_,[{5,leex,missing_code}]}],[]} = leex:file(Filename, Ret),
198
?line ok = file:write_file(Filename,
203
?line {error,[{_,[{4,leex,{regexp,_}}]}],[]} =
204
leex:file(Filename, Ret),
205
?line ok = file:write_file(Filename,
211
?line {error,[{_,[{5,leex,{regexp,_}}]}],[]} =
212
leex:file(Filename, Ret),
213
?line ok = file:write_file(Filename,
218
?line {error,[{_,[{4,leex,{regexp,_}}]}],[]} =
219
leex:file(Filename, Ret),
220
?line ok = file:write_file(Filename,
225
?line {error,[{_,[{5,leex,bad_rule}]}],[]} =
226
leex:file(Filename, Ret),
227
?line ok = file:write_file(Filename,
232
?line {error,[{_,[{4,leex,bad_rule}]}],[]} =
233
leex:file(Filename, Ret),
234
?line ok = file:write_file(Filename,
239
?line {error,[{_,[{4,erl_scan,_}]}],[]} = leex:file(Filename, Ret),
240
?line ok = file:write_file(Filename,
244
?line {error,[{_,[{3,leex,empty_rules}]}],[]} = leex:file(Filename, Ret),
245
?line ok = file:write_file(Filename,
250
?line {error,[{_,[{4,leex,empty_rules}]}],[]} = leex:file(Filename, Ret),
251
?line ok = file:write_file(Filename,
254
?line {error,[{_,[{2,leex,missing_rules}]}],[]} = leex:file(Filename, Ret),
255
?line ok = file:write_file(Filename,
259
?line {error,[{_,[{3,leex,missing_rules}]}],[]} = leex:file(Filename, Ret),
260
?line ok = file:write_file(Filename,
262
%% This is a weird line:
263
?line {error,[{_,[{0,leex,missing_defs}]}],[]} = leex:file(Filename, Ret),
264
?line ok = file:write_file(Filename,
266
?line {error,[{_,[{1,leex,missing_defs}]}],[]} = leex:file(Filename, Ret),
268
%% Check that correct line number is used in messages.
269
ErlFile = filename:join(Dir, "file.erl"),
270
Ret1 = [{scannerfile,ErlFile}|Ret],
271
?line ok = file:write_file(Filename,
276
" {word,TokenLine,TokenChars,\n"
277
" DDDD}}.\n" % unbound
279
"an error.\n">>), % syntax error
280
?line {ok, _, []} = leex:file(Filename, Ret1),
282
[{_,[{8,_,["syntax error before: ","error"]}]},
283
{_,[{6,_,{unbound_var,'DDDD'}}]}],
285
compile:file(ErlFile, [basic_validation, return]),
287
%% Ignored characters
288
?line ok = file:write_file(Filename,
289
<<"Definitions. D = [0-9]\n"
292
"Erlang code. f() -> a.\n">>),
294
[{1,leex,ignored_characters},
295
{2,leex,ignored_characters},
296
{4,leex,ignored_characters}]}]} =
297
leex:file(Filename, Ret),
299
?line ok = file:write_file(Filename,
303
"{L}+\\ : token.\n">>),
304
?line {error,[{_,[{4,leex,{regexp,{unterminated,"\\"}}}]}],[]} =
305
leex:file(Filename, Ret),
306
?line ok = file:write_file(Filename,
310
"{L}+\\x : token.\n">>),
311
?line {error,[{_,[{4,leex,{regexp,{illegal_char,"\\x"}}}]}],[]} =
312
leex:file(Filename, Ret),
313
?line ok = file:write_file(Filename,
317
"{L}+\\x{ : token.\n">>),
318
?line {error,[{_,[{4,leex,{regexp,{unterminated,"\\x{"}}}]}],[]} =
319
leex:file(Filename, Ret),
320
?line ok = file:write_file(Filename,
324
"[^ab : token.\n">>),
325
?line {error,[{_,[{4,leex,{regexp,{unterminated,"["}}}]}],[]} =
326
leex:file(Filename, Ret),
327
?line ok = file:write_file(Filename,
332
?line {error,[{_,[{4,leex,{regexp,{unterminated,"("}}}]}],[]} =
333
leex:file(Filename, Ret),
334
?line ok = file:write_file(Filename,
338
"[b-a] : token.\n">>),
339
?line {error,[{_,[{4,leex,{regexp,{char_class,"b-a"}}}]}],[]} =
340
leex:file(Filename, Ret),
342
?line ok = file:write_file(Filename,
346
"\\x{333333333333333333333333} : token.\n">>),
347
?line {error,[{_,[{4,leex,{regexp,
349
"\\x{333333333333333333333333}"}}}]}],[]} =
350
leex:file(Filename, Ret),
355
"Pushing back characters.";
357
pt(Config) when is_list(Config) ->
358
%% Needs more testing...
365
"{L}+ : {token,{word,TokenLine,TokenChars}}.\n"
366
"abc{D}+ : {skip_token,\"sture\" ++ string:substr(TokenChars, 4)}.\n"
367
"{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n"
369
"\\r\\n : {end_token,{crlf,TokenLine}}.\n"
374
{ok,[{word,1,\"sture\"},{integer,1,123}],1} =
375
string(\"abc123\"), ok. ">>,
379
?line run(Config, Ts),
383
"Examples from the manpage.";
385
man(Config) when is_list(Config) ->
389
"[a-z][0-9a-zA-Z_]* :\n"
390
" {token,{atom,TokenLine,list_to_atom(TokenChars)}}.\n"
391
"[A-Z_][0-9a-zA-Z_]* :\n"
392
" {token,{var,TokenLine,list_to_atom(TokenChars)}}.\n"
393
"(\\+|-)?[0-9]+\\.[0-9]+((E|e)(\\+|-)?[0-9]+)? : \n"
394
" {token,{float,TokenLine,list_to_float(TokenChars)}}.\n"
395
"\\s : skip_token.\n"
399
" {ok,[{float,1,3.14},{atom,1,atom},{var,1,'V314'}],1} =\n"
400
" string(\"3.14atom V314\"),\n"
410
" {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n"
411
"{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? :\n"
412
" {token,{float,TokenLine,list_to_float(TokenChars)}}.\n"
413
"\\s : skip_token.\n"
417
" {ok,[{float,1,3.14},{integer,1,314}],1} = \n"
418
" string(\"3.14 314\"),\n"
423
?line run(Config, Ts),
429
ex(Config) when is_list(Config) ->
432
"D = [0-543-705-982]\n"
435
" {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n"
437
" {token,{list_to_atom(TokenChars),TokenLine}}.\n"
441
" {ok,[{integer,1,12},{' c\\na',1},{integer,2,34},{b789a,2}],2} =\n"
442
" string(\"12 c\\na34b789a\"),\n"
452
"{L}+ : {token,chars}.\n"
453
"zyx{D}+ : {token,zyx}.\n"
454
"\\s : skip_token.\n"
458
" {ok,[chars,zyx],1} = string(\"abcdef zyx123\"),\n"
467
"{NL}* : {token,newlines}.\n"
471
" {ok,[],1} = string(\"\"), ok.\n">>, % string("a") would loop...
478
"SP0 = [\\000-\\n]\n"
480
"{SP0}+ : {token,{small,TokenChars}}.\n"
481
"{SP1}+ : {token,{big,TokenChars}}.\n"
485
" string(\"\\x00\\n\\s\\n\\n\"),\n"
493
"W = [\\s\\b\\n\\r\\t\\e\\v\\d\\f]\n"
495
"\\[{L}+(,{L}+)*\\] : {token,{list,TokenChars}}.\n"
496
"\"{L}+\" : {token,{string,TokenChars}}.\n"
497
"\\$. : {token,{char,TokenChars}}.\n"
498
"{W}+ : {token,{white,TokenChars}}.\n"
499
"ff\\f+ : {token,{form,TokenChars}}.\n"
500
"\\$\\^+\\\\+ : {token,{other,TokenChars}}.\n"
504
" {ok,[{white,\"\\b\\f\"}],1} = string(\"\\b\\f\"),\n"
505
" {ok,[{form,\"ff\\f\"}],1} = string(\"ff\\f\"),\n"
506
" {ok,[{string,\"\\\"foo\\\"\"}],1} = string(\"\\\"foo\\\"\"),\n"
507
" {ok,[{char,\"$.\"}],1} = string(\"$\\.\"),\n"
508
" {ok,[{list,\"[a,b,c]\"}],1} = string(\"[a,b,c]\"),\n"
509
" {ok,[{other,\"$^\\\\\"}],1} = string(\"$^\\\\\"),\n"
518
"L}+ : {token,{TokenChars,#r.f}}.\n"
523
" string(\"abc\"),\n"
528
{ex_7, %% Assumes regexp can handle \x
530
"H1 = \\x11\\x{ab}\n"
531
"H2 = [\\x{30}\\x{ac}]\n"
533
"{H1}{H2}+ : {token,{hex,TokenChars}}.\n"
537
" {ok,[{hex,[17,171,48,172]}],1} =\n"
538
" string(\"\\x{11}\\xab0\\xac\"),\n"
543
?line run(Config, Ts),
549
ex2(Config) when is_list(Config) ->
552
%%% File : erlang_scan.xrl
553
%%% Author : Robert Virding
554
%%% Purpose : Tkoen definitions for Erlang.
562
A = ({U}|{L}|{D}|_|@)
563
WS = ([\\000-\\s]|%.*)
566
{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? :
567
{token,{float,TokenLine,list_to_float(TokenChars)}}.
568
{D}+#{H}+ : base(TokenLine, TokenChars).
569
{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}.
570
{L}{A}* : Atom = list_to_atom(TokenChars),
571
{token,case reserved_word(Atom) of
572
true -> {Atom,TokenLine};
573
false -> {atom,TokenLine,Atom}
575
'(\\\\\\^.|\\\\.|[^'])*' :
577
S = lists:sublist(TokenChars, 2, TokenLen - 2),
578
case catch list_to_atom(string_gen(S)) of
579
{'EXIT',_} -> {error,\"illegal atom \" ++ TokenChars};
580
Atom -> {token,{atom,TokenLine,Atom}}
582
({U}|_){A}* : {token,{var,TokenLine,list_to_atom(TokenChars)}}.
583
\"(\\\\\\^.|\\\\.|[^\"])*\" :
585
S = lists:sublist(TokenChars, 2, TokenLen - 2),
586
{token,{string,TokenLine,string_gen(S)}}.
587
\\$(\\\\{O}{O}{O}|\\\\\\^.|\\\\.|.) :
588
{token,{char,TokenLine,cc_convert(TokenChars)}}.
589
-> : {token,{'->',TokenLine}}.
590
:- : {token,{':-',TokenLine}}.
591
\\|\\| : {token,{'||',TokenLine}}.
592
<- : {token,{'<-',TokenLine}}.
593
\\+\\+ : {token,{'++',TokenLine}}.
594
-- : {token,{'--',TokenLine}}.
595
=/= : {token,{'=/=',TokenLine}}.
596
== : {token,{'==',TokenLine}}.
597
=:= : {token,{'=:=',TokenLine}}.
598
/= : {token,{'/=',TokenLine}}.
599
>= : {token,{'>=',TokenLine}}.
600
=< : {token,{'=<',TokenLine}}.
601
<= : {token,{'<=',TokenLine}}.
602
<< : {token,{'<<',TokenLine}}.
603
>> : {token,{'>>',TokenLine}}.
604
:: : {token,{'::',TokenLine}}.
605
[]()[}{|!?/;:,.*+#<>=-] :
606
{token,{list_to_atom(TokenChars),TokenLine}}.
607
\\.{WS} : {end_token,{dot,TokenLine}}.
612
-export([reserved_word/1]).
614
%% reserved_word(Atom) -> Bool
615
%% return 'true' if Atom is an Erlang reserved word, else 'false'.
617
reserved_word('after') -> true;
618
reserved_word('begin') -> true;
619
reserved_word('case') -> true;
620
reserved_word('try') -> true;
621
reserved_word('cond') -> true;
622
reserved_word('catch') -> true;
623
reserved_word('andalso') -> true;
624
reserved_word('orelse') -> true;
625
reserved_word('end') -> true;
626
reserved_word('fun') -> true;
627
reserved_word('if') -> true;
628
reserved_word('let') -> true;
629
reserved_word('of') -> true;
630
reserved_word('query') -> true;
631
reserved_word('receive') -> true;
632
reserved_word('when') -> true;
633
reserved_word('bnot') -> true;
634
reserved_word('not') -> true;
635
reserved_word('div') -> true;
636
reserved_word('rem') -> true;
637
reserved_word('band') -> true;
638
reserved_word('and') -> true;
639
reserved_word('bor') -> true;
640
reserved_word('bxor') -> true;
641
reserved_word('bsl') -> true;
642
reserved_word('bsr') -> true;
643
reserved_word('or') -> true;
644
reserved_word('xor') -> true;
645
reserved_word('spec') -> true;
646
reserved_word(_) -> false.
649
H = string:chr(Cs, $#),
650
case list_to_integer(string:substr(Cs, 1, H-1)) of
651
B when B > 16 -> {error,\"illegal base\"};
653
case base(string:substr(Cs, H+1), B, 0) of
654
error -> {error,\"illegal based number\"};
655
N -> {token,{integer,L,N}}
659
base([C|Cs], Base, SoFar) when C >= $0, C =< $9, C < Base + $0 ->
660
Next = SoFar * Base + (C - $0),
661
base(Cs, Base, Next);
662
base([C|Cs], Base, SoFar) when C >= $a, C =< $f, C < Base + $a - 10 ->
663
Next = SoFar * Base + (C - $a + 10),
664
base(Cs, Base, Next);
665
base([C|Cs], Base, SoFar) when C >= $A, C =< $F, C < Base + $A - 10 ->
666
Next = SoFar * Base + (C - $A + 10),
667
base(Cs, Base, Next);
668
base([_|_], _, _) -> error; %Unknown character
671
cc_convert([$$,$\\\\|Cs]) ->
672
hd(string_escape(Cs));
673
cc_convert([$$,C]) -> C.
675
string_gen([$\\\\|Cs]) ->
677
string_gen([C|Cs]) ->
679
string_gen([]) -> [].
681
string_escape([O1,O2,O3|S]) when
682
O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 ->
683
[(O1*8 + O2)*8 + O3 - 73*$0|string_gen(S)];
684
string_escape([$^,C|Cs]) ->
685
[C band 31|string_gen(Cs)];
686
string_escape([C|Cs]) when C >= $\\000, C =< $\\s ->
688
string_escape([C|Cs]) ->
689
[escape_char(C)|string_gen(Cs)].
691
escape_char($n) -> $\\n; %\\n = LF
692
escape_char($r) -> $\\r; %\\r = CR
693
escape_char($t) -> $\\t; %\\t = TAB
694
escape_char($v) -> $\\v; %\\v = VT
695
escape_char($b) -> $\\b; %\\b = BS
696
escape_char($f) -> $\\f; %\\f = FF
697
escape_char($e) -> $\\e; %\\e = ESC
698
escape_char($s) -> $\\s; %\\s = SPC
699
escape_char($d) -> $\\d; %\\d = DEL
703
XrlFile = filename:join(Dir, "erlang_scan.xrl"),
704
?line ok = file:write_file(XrlFile, Xrl),
705
ErlFile = filename:join(Dir, "erlang_scan.erl"),
706
?line {ok, _} = leex:file(XrlFile, []),
707
?line {ok, _} = compile:file(ErlFile, [{outdir,Dir}]),
708
code:purge(erlang_scan),
709
AbsFile = filename:rootname(ErlFile, ".erl"),
710
code:load_abs(AbsFile, erlang_scan),
712
F = fun(Cont, Chars, Location) ->
713
erlang_scan:tokens(Cont, Chars, Location)
715
F1 = fun(Cont, Chars, Location) ->
716
erlang_scan:token(Cont, Chars, Location)
720
{ok, Ts, 1} = scan_tokens_1(S, F, 1),
721
{ok, Ts, 1} = scan_token_1(S, F1, 1),
722
{ok, Ts, 1} = scan_tokens(S, F, 1),
723
{ok, Ts, 1} = erlang_scan:string(S, 1)
727
{ok, Ts, 2} = scan_tokens_1(S, F, 1),
728
{ok, Ts, 2} = scan_token_1(S, F1, 1),
729
{ok, Ts, 2} = scan_tokens(S, F, 1),
730
{ok, Ts, 2} = erlang_scan:string(S, 1)
734
{ok, Ts, 1} = scan_tokens_1(S, F, 1),
735
{ok, Ts, 1} = scan_token_1(S, F1, 1),
736
{ok, Ts, 1} = scan_tokens(S, F, 1),
737
{ok, Ts, 1} = erlang_scan:string(S, 1)
739
{ok,[{integer,1,99},{dot,1}],1} = erlang_scan:string("99. "),
741
Atom = "'" ++ lists:duplicate(1000,$a) ++ "'",
743
Reason = "illegal atom " ++ Atom,
744
Err = {error,{1,erlang_scan,{user,Reason}},1},
745
{done,Err,[]} = scan_tokens_1(S, F, 1),
746
{done,Err,[]} = scan_token_1(S, F1, 1),
747
{done,Err,[]} = scan_tokens(S, F, 1),
748
Err = erlang_scan:string(S, 1)
752
Err = {error,{1,erlang_scan,{illegal,[2730]}},1},
753
{done,Err,[]} = scan_tokens_1(S, F, 1),
754
{done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty
755
{done,Err,[]} = scan_tokens(S, F, 1),
756
Err = erlang_scan:string(S, 1)
759
S = "\x{aaa} + 1. 34",
760
Err = {error,{1,erlang_scan,{illegal,[2730]}},1},
761
{done,Err,[]} = scan_tokens_1(S, F, 1),
762
{done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty
763
{done,Err,"34"} = scan_tokens(S, F, 1),
764
Err = erlang_scan:string(S, 1)
767
S = "\x{aaa} \x{bbb}. 34",
768
Err = {error,{1,erlang_scan,{illegal,[2730]}},1},
769
{done,Err,[]} = scan_tokens_1(S, F, 1),
770
{done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty
771
{done,Err,"34"} = scan_tokens(S, F, 1),
772
Err = erlang_scan:string(S, 1)
775
S = "\x{aaa} 18#34. 34",
776
Err = {error,{1,erlang_scan,{illegal,[2730]}},1},
777
{done,Err,[]} = scan_tokens_1(S, F, 1),
778
{done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty
779
{done,Err,"34"} = scan_tokens(S, F, 1),
780
Err = erlang_scan:string(S, 1)
784
Err = {error,{1,erlang_scan,{illegal,[2730]}},1},
785
{done,Err,eof} = scan_tokens_1(S, F, 1),
786
{done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty
787
{done,Err,eof} = scan_tokens(S, F, 1),
788
Err = erlang_scan:string(S, 1)
792
scan_tokens(String, Fun, Location) ->
793
scan_tokens(String, Fun, Location, []).
795
scan_tokens(String, Fun, Location, Rs) ->
796
case Fun([], String, Location) of
797
{done, {error,_,_}, _} = Error ->
799
{done, {ok,Ts,End}, ""} ->
800
{ok, lists:append(lists:reverse([Ts|Rs])), End};
801
{done, {ok,Ts,End}, Rest} ->
802
scan_tokens(Rest, Fun, End, [Ts|Rs])
805
scan_tokens_1(String, Fun, Location) ->
806
scan_tokens_1({more, []}, String, Fun, Location, []).
808
scan_tokens_1({done, {error, _, _}, _}=Error, _Cs, _Fun, _Location, _Rs) ->
810
scan_tokens_1({done, {ok,Ts,End}, ""}, "", _Fun, _Location, Rs) ->
811
{ok,lists:append(lists:reverse([Ts|Rs])),End};
812
scan_tokens_1({done, {ok,Ts,End}, Rest}, Cs, Fun, _Location, Rs) ->
813
scan_tokens_1({more,[]}, Rest++Cs, Fun, End, [Ts|Rs]);
814
scan_tokens_1({more, Cont}, [C | Cs], Fun, Loc, Rs) ->
815
R = Fun(Cont, [C], Loc),
816
scan_tokens_1(R, Cs, Fun, Loc, Rs);
817
scan_tokens_1({more, Cont}, eof, Fun, Loc, Rs) ->
818
R = Fun(Cont, eof, Loc),
819
scan_tokens_1(R, eof, Fun, Loc, Rs).
821
scan_token_1(String, Fun, Location) ->
822
scan_token_1({more, []}, String, Fun, Location, []).
824
scan_token_1({done, {error, _, _}, _}=Error, _Cs, _Fun, _Location, _Rs) ->
826
scan_token_1({done, {ok,Ts,End}, ""}, "", _Fun, _Location, Rs) ->
827
{ok,lists:reverse([Ts|Rs]),End};
828
scan_token_1({done, {ok,Ts,End}, Rest}, Cs, Fun, _Location, Rs) ->
829
scan_token_1({more,[]}, Rest++Cs, Fun, End, [Ts|Rs]);
830
scan_token_1({more, Cont}, [C | Cs], Fun, Loc, Rs) ->
831
R = Fun(Cont, [C], Loc),
832
scan_token_1(R, Cs, Fun, Loc, Rs).
837
"Not yet implemented.";
838
not_yet(suite) -> [];
839
not_yet(Config) when is_list(Config) ->
841
Filename = filename:join(Dir, "file.xrl"),
842
Ret = [return, {report, true}],
843
?line ok = file:write_file(Filename,
848
?line {error,[{_,[{3,leex,{regexp,_}}]}],[]} =
849
leex:file(Filename, Ret),
850
?line ok = file:write_file(Filename,
855
?line {error,[{_,[{3,leex,{regexp,_}}]}],[]} =
856
leex:file(Filename, Ret),
861
{ok, Info} = file:read_file_info(Fname),
862
Mode = Info#file_info.mode - 8#00200,
863
ok = file:write_file_info(Fname, Info#file_info{mode = Mode}).
866
{ok, Info} = file:read_file_info(Fname),
867
Mode = Info#file_info.mode bor 8#00200,
868
ok = file:write_file_info(Fname, Info#file_info{mode = Mode}).
870
run(Config, Tests) ->
871
F = fun({N,P,Pre,E}) ->
872
case catch run_test(Config, P, Pre) of
876
?t:format("~nTest ~p failed. Expected~n ~p~n"
877
"but got~n ~p~n", [N, E, Bad]),
881
lists:foreach(F, Tests).
883
run_test(Config, Def, Pre) ->
884
%% io:format("testing ~s~n", [binary_to_list(Def)]),
885
DefFile = 'leex_test.xrl',
886
Filename = 'leex_test.erl',
888
XrlFile = filename:join(DataDir, DefFile),
889
ErlFile = filename:join(DataDir, Filename),
890
Opts = [return, warn_unused_vars,{outdir,DataDir}],
891
ok = file:write_file(XrlFile, Def),
892
LOpts = [return, {report, false} |
899
XOpts = [verbose, dfa_graph], % just to get some code coverage...
900
LRet = leex:file(XrlFile, XOpts ++ LOpts),
902
{ok, _Outfile, _LWs} ->
903
CRet = compile:file(ErlFile, Opts),
906
AbsFile = filename:rootname(ErlFile, ".erl"),
909
code:load_abs(AbsFile, Mod),
911
%% warnings(ErlFile, Ws);
912
{error, [{ErlFile,Es}], []} -> {error, Es, []};
913
{error, [{ErlFile,Es}], [{ErlFile,Ws}]} -> {error, Es, Ws};
916
{error, [{XrlFile,LEs}], []} -> {error, LEs, []};
917
{error, [{XrlFile,LEs}], [{XrlFile,LWs}]} -> {error, LEs, LWs};
921
extract(File, {error, Es, Ws}) ->
922
{errors, extract(File, Es), extract(File, Ws)};
924
lists:append([T || {F, T} <- Ts, F =:= File]).