217
211
(* could assert pinfo.filename = file ? *)
218
212
match Ast_c.pinfo_of_info ii with
219
213
Ast_c.OriginTok pi ->
220
Ast_c.OriginTok (Common.complete_parse_info file table pi)
214
Ast_c.OriginTok (Common.complete_parse_info_large file table pi)
221
215
| Ast_c.ExpandedTok (pi,vpi) ->
222
Ast_c.ExpandedTok((Common.complete_parse_info file table pi),vpi)
216
Ast_c.ExpandedTok((Common.complete_parse_info_large file table pi),vpi)
223
217
| Ast_c.FakeTok (s,vpi) -> Ast_c.FakeTok (s,vpi)
224
218
| Ast_c.AbstractLineTok pi -> failwith "should not occur"
494
488
Visitor_c.kexpr_s = (fun (k, bigf) x ->
489
match Ast_c.get_e_and_ii x with
496
490
| (Ast_c.SizeOfExpr e, tref), isizeof ->
497
491
let i1 = tuple_of_list1 isizeof in
492
(match Ast_c.get_e_and_ii e with
499
493
| (Ast_c.ParenExpr e, _), iiparen ->
494
let (i2, i3) = tuple_of_list2 iiparen in
495
(match Ast_c.get_e_and_ii e with
501
496
| (Ast_c.Ident (ident), _), _ii ->
503
498
let s = Ast_c.str_of_name ident in
504
499
if List.mem s !ident_to_type
506
501
let t = ident_to_typename ident in
507
let (i2, i3) = tuple_of_list2 iiparen in
508
(Ast_c.SizeOfType t, tref), [i1;i2;i3]
502
(Ast_c.SizeOfType t, tref),[i1;i2;i3]
528
522
(* Error recovery *)
529
523
(*****************************************************************************)
525
let is_define_passed passed =
526
let xs = passed +> List.rev +> List.filter TH.is_not_comment in
527
if List.length xs >= 2
529
(match Common.head_middle_tail xs with
530
| Parser_c.TDefine _, _, Parser_c.TDefEOL _ ->
535
pr2_err "WEIRD: length list of error recovery tokens < 2 ";
539
let is_defined_passed_bis last_round =
540
let xs = last_round +> List.filter TH.is_not_comment in
542
| Parser_c.TDefine _::_ -> true
545
(* ---------------------------------------------------------------------- *)
531
548
(* todo: do something if find Parser_c.Eof ? *)
532
549
let rec find_next_synchro next already_passed =
591
602
and find_next_synchro_orig next already_passed =
594
pr2 "ERROR-RECOV: end of file while in recovery mode";
605
pr2_err "ERROR-RECOV: end of file while in recovery mode";
595
606
already_passed, []
597
608
| (Parser_c.TCBrace i as v)::xs when TH.col_of_tok v =|= 0 ->
598
pr2 ("ERROR-RECOV: found sync '}' at line "^i_to_s (TH.line_of_tok v));
609
pr2_err ("ERROR-RECOV: found sync '}' at line "^i_to_s (TH.line_of_tok v));
601
612
| [] -> raise Impossible (* there is a EOF token normally *)
603
614
(* still useful: now parser.mly allow empty ';' so normally no pb *)
604
615
| Parser_c.TPtVirg iptvirg::xs ->
605
pr2 "ERROR-RECOV: found sync bis, eating } and ;";
616
pr2_err "ERROR-RECOV: found sync bis, eating } and ;";
606
617
(Parser_c.TPtVirg iptvirg)::v::already_passed, xs
608
619
| Parser_c.TIdent x::Parser_c.TPtVirg iptvirg::xs ->
609
pr2 "ERROR-RECOV: found sync bis, eating ident, }, and ;";
620
pr2_err "ERROR-RECOV: found sync bis, eating ident, }, and ;";
610
621
(Parser_c.TPtVirg iptvirg)::(Parser_c.TIdent x)::v::already_passed,
613
624
| Parser_c.TCommentSpace sp::Parser_c.TIdent x::Parser_c.TPtVirg iptvirg
615
pr2 "ERROR-RECOV: found sync bis, eating ident, }, and ;";
626
pr2_err "ERROR-RECOV: found sync bis, eating ident, }, and ;";
616
627
(Parser_c.TCommentSpace sp)::
617
628
(Parser_c.TPtVirg iptvirg)::
618
629
(Parser_c.TIdent x)::
634
645
v::already_passed, xs
636
647
| v::xs when TH.col_of_tok v =|= 0 && TH.is_start_of_something v ->
637
pr2 ("ERROR-RECOV: found sync col 0 at line "^ i_to_s(TH.line_of_tok v));
648
pr2_err ("ERROR-RECOV: found sync col 0 at line "^ i_to_s(TH.line_of_tok v));
638
649
already_passed, v::xs
641
652
find_next_synchro_orig xs (v::already_passed)
655
(*****************************************************************************)
656
(* Macro problem recovery *)
657
(*****************************************************************************)
658
module TV = Token_views_c
660
let candidate_macros_in_passed2 passed defs_optional =
664
passed +> List.iter (function
665
| Parser_c.TIdent (s,_)
666
(* bugfix: may have to undo some infered things *)
667
| Parser_c.TMacroIterator (s,_)
668
| Parser_c.TypedefIdent (s,_)
670
(match Common.hfind_option s defs_optional with
672
if s ==~ Parsing_hacks.regexp_macro
674
(* pr2 (spf "candidate: %s" s); *)
675
Common.push2 (s, def) res
677
Common.push2 (s, def) res2
687
let candidate_macros_in_passed a b =
688
Common.profile_code "MACRO managment" (fun () ->
689
candidate_macros_in_passed2 a b)
693
let find_optional_macro_to_expand2 ~defs toks =
695
let defs = Common.hash_of_list defs in
697
let toks = toks +> Common.map (function
699
(* special cases to undo *)
700
| Parser_c.TMacroIterator (s, ii) ->
701
if Hashtbl.mem defs s
702
then Parser_c.TIdent (s, ii)
703
else Parser_c.TMacroIterator (s, ii)
705
| Parser_c.TypedefIdent (s, ii) ->
706
if Hashtbl.mem defs s
707
then Parser_c.TIdent (s, ii)
708
else Parser_c.TypedefIdent (s, ii)
714
Parsing_hacks.fix_tokens_cpp ~macro_defs:defs tokens
716
(* just calling apply_macro_defs and having a specialized version
717
* of the code in fix_tokens_cpp is not enough as some work such
718
* as the passing of the body of attribute in Parsing_hacks.find_macro_paren
719
* will not get the chance to be run on the new expanded tokens.
720
* Hence even if it's expensive, it's currently better to
721
* just call directly fix_tokens_cpp again here.
723
let tokens2 = ref (tokens +> Common.acc_map TV.mk_token_extended) in
724
let cleaner = !tokens2 +> Parsing_hacks.filter_cpp_stuff in
725
let paren_grouped = TV.mk_parenthised cleaner in
726
Cpp_token_c.apply_macro_defs
727
~msg_apply_known_macro:(fun s -> pr2 (spf "APPLYING: %s" s))
728
~msg_apply_known_macro_hint:(fun s -> pr2 "hint")
730
(* because the before field is used by apply_macro_defs *)
731
tokens2 := TV.rebuild_tokens_extented !tokens2;
732
Parsing_hacks.insert_virtual_positions
733
(!tokens2 +> Common.acc_map (fun x -> x.TV.tok))
735
let find_optional_macro_to_expand ~defs a =
736
Common.profile_code "MACRO managment" (fun () ->
737
find_optional_macro_to_expand2 ~defs a)
644
741
(*****************************************************************************)
645
742
(* Include/Define hacks *)
646
743
(*****************************************************************************)
716
815
(*****************************************************************************)
718
817
let parse_cpp_define_file2 file =
719
let toks = tokens ~profile:false file in
720
let toks = Parsing_hacks.fix_tokens_define toks in
721
Parsing_hacks.extract_cpp_define toks
818
Common.save_excursion Flag_parsing_c.verbose_lexing (fun () ->
819
Flag_parsing_c.verbose_lexing := false;
820
let toks = tokens ~profile:false file in
821
let toks = Cpp_token_c.fix_tokens_define toks in
822
Cpp_token_c.extract_cpp_define toks
723
825
let parse_cpp_define_file a =
724
826
Common.profile_code_exclusif "HACK" (fun () -> parse_cpp_define_file2 a)
830
let (_defs : (string, Cpp_token_c.define_def) Hashtbl.t ref) =
831
ref (Hashtbl.create 101)
833
let (_defs_builtins : (string, Cpp_token_c.define_def) Hashtbl.t ref) =
834
ref (Hashtbl.create 101)
726
837
(* can not be put in parsing_hack, cos then mutually recursive problem as
727
838
* we also want to parse the standard.h file.
729
let init_defs std_h =
840
let init_defs_macros std_h =
730
841
if not (Common.lfile_exists std_h)
731
842
then pr2 ("warning: Can't find default macro file: " ^ std_h)
733
844
pr2 ("init_defs: " ^ std_h);
734
Parsing_hacks._defs := Common.hash_of_list (parse_cpp_define_file std_h);
845
_defs := Common.hash_of_list (parse_cpp_define_file std_h);
848
let init_defs_builtins file_h =
849
if not (Common.lfile_exists file_h)
850
then pr2 ("warning: Can't find macro file: " ^ file_h)
852
pr2 ("init_defs_builtins: " ^ file_h);
854
Common.hash_of_list (parse_cpp_define_file file_h);
804
924
mutable passed : Parser_c.token list;
805
925
mutable passed_clean : Parser_c.token list;
807
let clone_tokens_stat tr =
928
let mk_tokens_state toks =
931
rest_clean = (toks +> List.filter TH.is_not_comment);
932
current = (List.hd toks);
939
let clone_tokens_state tr =
808
940
{ rest = tr.rest;
809
941
rest_clean = tr.rest_clean;
810
942
current = tr.current;
811
943
passed = tr.passed;
812
944
passed_clean = tr.passed_clean;
814
let copy_tokens_stat ~src ~dst =
946
let copy_tokens_state ~src ~dst =
815
947
dst.rest <- src.rest;
816
948
dst.rest_clean <- src.rest_clean;
817
949
dst.current <- src.current;
971
1106
Common.profile_code_exclusif "YACC" (fun () ->
972
1107
Left (Parser_c.celem (lexer_function ~pass tr) lexbuf_fake)
975
if (pass =|= 1 && !Flag_parsing_c.disable_two_pass)|| (pass =|= 2)
978
(* Lexical is not anymore launched I think *)
979
| Lexer_c.Lexical s ->
980
pr2 ("lexical error " ^s^ "\n =" ^ error_msg_tok tr.current)
981
| Parsing.Parse_error ->
982
pr2 ("parse error \n = " ^ error_msg_tok tr.current)
983
| Semantic_c.Semantic (s, i) ->
984
pr2 ("semantic error " ^s^ "\n ="^ error_msg_tok tr.current)
988
1110
LP.restore_typedef_state();
990
1112
(* must keep here, before the code that adjusts the tr fields *)
991
1113
let line_error = TH.line_of_tok tr.current in
1115
let passed_before_error = tr.passed in
1116
let current = tr.current in
994
1118
(* error recovery, go to next synchro point *)
995
1119
let (passed', rest') = find_next_synchro tr.rest tr.passed in
1032
1157
(* -------------------------------------------------- *)
1033
1158
LP.lexer_reset_typedef();
1034
1159
Parsing_hacks.ifdef_paren_cnt := 0;
1035
1161
let toks_orig = tokens file in
1037
let toks = Parsing_hacks.fix_tokens_define toks_orig in
1038
let toks = Parsing_hacks.fix_tokens_cpp toks in
1042
rest_clean = (toks +> List.filter TH.is_not_comment);
1043
current = (List.hd toks);
1163
let toks = Cpp_token_c.fix_tokens_define toks_orig in
1165
let toks = Parsing_hacks.fix_tokens_cpp ~macro_defs:!_defs_builtins toks in
1167
(* expand macros on demand trick, preparation phase *)
1169
Common.profile_code "MACRO mgmt prep 1" (fun () ->
1170
let macros = Hashtbl.copy !_defs in
1171
(* include also builtins as some macros may generate some builtins too
1172
* like __decl_spec or __stdcall
1174
!_defs_builtins +> Hashtbl.iter (fun s def ->
1175
Hashtbl.replace macros s def;
1180
Common.profile_code "MACRO mgmt prep 2" (fun () ->
1181
let local_macros = parse_cpp_define_file file in
1182
local_macros +> List.iter (fun (s, def) ->
1183
Hashtbl.replace macros s def;
1187
let tr = mk_tokens_state toks in
1051
1189
let rec loop tr =
1063
1201
let checkpoint = TH.line_of_tok tr.current in
1064
1202
let checkpoint_file = TH.file_of_tok tr.current in
1066
let tr_save = clone_tokens_stat tr in
1068
1204
(* call the parser *)
1070
let pass1 = get_one_elem ~pass:1 tr (file, filelines) in
1207
Common.profile_code "Parsing: 1st pass" (fun () ->
1208
get_one_elem ~pass:1 tr (file, filelines)
1071
1210
match pass1 with
1072
1211
| Left e -> Left e
1074
if !Flag_parsing_c.disable_two_pass
1212
| Right (info,line_err, passed, passed_before_error, cur, exn) ->
1213
if !Flag_parsing_c.disable_multi_pass
1077
pr2 "parsing pass2: try again";
1078
copy_tokens_stat ~src:tr_save ~dst: tr;
1079
let pass2 = get_one_elem ~pass:2 tr (file, filelines) in
1216
Common.profile_code "Parsing: multi pass" (fun () ->
1218
pr2_err "parsing pass2: try again";
1219
let toks = List.rev passed ++ tr.rest in
1220
let new_tr = mk_tokens_state toks in
1221
copy_tokens_state ~src:new_tr ~dst:tr;
1222
let passx = get_one_elem ~pass:2 tr (file, filelines) in
1226
| Right (info,line_err,passed,passed_before_error,cur,exn) ->
1228
candidate_macros_in_passed passed macros
1230
if is_define_passed passed || null candidates
1233
(* todo factorize code *)
1235
pr2_err "parsing pass3: try again";
1236
let toks = List.rev passed ++ tr.rest in
1238
find_optional_macro_to_expand ~defs:candidates toks in
1239
let new_tr = mk_tokens_state toks' in
1240
copy_tokens_state ~src:new_tr ~dst:tr;
1241
let passx = get_one_elem ~pass:3 tr (file, filelines) in
1245
| Right (info,line_err,passed,passed_before_error,cur,exn) ->
1246
pr2_err "parsing pass4: try again";
1249
candidate_macros_in_passed passed macros in
1251
let toks = List.rev passed ++ tr.rest in
1253
find_optional_macro_to_expand ~defs:candidates toks in
1254
let new_tr = mk_tokens_state toks' in
1255
copy_tokens_state ~src:new_tr ~dst:tr;
1256
let passx = get_one_elem ~pass:4 tr (file, filelines) in
1086
1267
let checkpoint2 = TH.line_of_tok tr.current in (* <> line_error *)
1087
1268
let checkpoint2_file = TH.file_of_tok tr.current in
1092
| Right (_, line_error, _) ->
1094
let xs = tr.passed +> List.rev +> List.filter TH.is_not_comment in
1095
if List.length xs >= 2
1097
(match Common.head_middle_tail xs with
1098
| Parser_c.TDefine _, _, Parser_c.TDefEOL _ ->
1103
pr2 "WEIRD: length list of error recovery tokens < 2 ";
1107
(if was_define && !Flag_parsing_c.filter_msg_define_error
1111
if (checkpoint_file =$= checkpoint2_file) &&
1112
checkpoint_file =$= file
1113
then print_bad line_error (checkpoint, checkpoint2) filelines
1114
else pr2 "PB: bad: but on tokens not from original file"
1121
1271
if (checkpoint_file =$= checkpoint2_file) && (checkpoint_file =$= file)
1122
1272
then (checkpoint2 - checkpoint)
1140
1290
stat.Stat.correct <- stat.Stat.correct + diffline;
1142
| Right (info_of_bads, line_error, toks_of_bads) ->
1292
| Right (info_of_bads, line_error, toks_of_bads,
1293
_passed_before_error, cur, exn) ->
1295
let was_define = is_define_passed tr.passed in
1297
if was_define && !Flag_parsing_c.filter_msg_define_error
1303
| Parsing.Parse_error
1304
| Semantic_c.Semantic _ -> ()
1308
if !Flag_parsing_c.show_parsing_error
1311
(* Lexical is not anymore launched I think *)
1312
| Lexer_c.Lexical s ->
1313
pr2 ("lexical error " ^s^ "\n =" ^ error_msg_tok cur)
1314
| Parsing.Parse_error ->
1315
pr2 ("parse error \n = " ^ error_msg_tok cur)
1316
| Semantic_c.Semantic (s, i) ->
1317
pr2 ("semantic error " ^s^ "\n ="^ error_msg_tok cur)
1318
| e -> raise Impossible
1321
if (checkpoint_file =$= checkpoint2_file) &&
1322
checkpoint_file =$= file
1323
then print_bad line_error (checkpoint, checkpoint2) filelines
1324
else pr2 "PB: bad: but on tokens not from original file"
1330
+> Common.filter (TH.is_same_line_or_close line_error)
1331
+> Common.filter TH.is_ident_like
1334
(pbline +> List.map TH.str_of_tok), line_error
1336
stat.Stat.problematic_lines <-
1337
error_info::stat.Stat.problematic_lines;
1143
1341
if was_define && !Flag_parsing_c.filter_define_error
1144
1342
then stat.Stat.correct <- stat.Stat.correct + diffline
1145
1343
else stat.Stat.bad <- stat.Stat.bad + diffline;
1149
+> Common.filter (TH.is_same_line_or_close line_error)
1150
+> Common.filter TH.is_ident_like
1153
(pbline +> List.map TH.str_of_tok), line_error
1155
stat.Stat.problematic_lines <-
1156
error_info::stat.Stat.problematic_lines;
1158
1345
Ast_c.NotParsedCorrectly info_of_bads
1215
1402
(*****************************************************************************)
1217
1404
let (cstatement_of_string: string -> Ast_c.statement) = fun s ->
1218
Common.write_file ("/tmp/__cocci.c") ("void main() { \n" ^ s ^ "\n}");
1219
let program = parse_c_and_cpp ("/tmp/__cocci.c") +> fst in
1405
let tmpfile = Common.new_temp_file "cocci_stmt_of_s" "c" in
1406
Common.write_file tmpfile ("void main() { \n" ^ s ^ "\n}");
1407
let program = parse_c_and_cpp tmpfile +> fst in
1220
1408
program +> Common.find_some (fun (e,_) ->
1222
1410
| Ast_c.Definition ({Ast_c.f_body = [Ast_c.StmtElem st]},_) -> Some st
1226
1414
let (cexpression_of_string: string -> Ast_c.expression) = fun s ->
1227
Common.write_file ("/tmp/__cocci.c") ("void main() { \n" ^ s ^ ";\n}");
1228
let program = parse_c_and_cpp ("/tmp/__cocci.c") +> fst in
1415
let tmpfile = Common.new_temp_file "cocci_expr_of_s" "c" in
1416
Common.write_file tmpfile ("void main() { \n" ^ s ^ ";\n}");
1417
let program = parse_c_and_cpp tmpfile +> fst in
1229
1418
program +> Common.find_some (fun (e,_) ->
1231
1420
| Ast_c.Definition ({Ast_c.f_body = compound},_) ->
1232
1421
(match compound with
1233
| [Ast_c.StmtElem (Ast_c.ExprStatement (Some e),ii)] -> Some e
1422
| [Ast_c.StmtElem st] ->
1423
(match Ast_c.unwrap_st st with
1424
| Ast_c.ExprStatement (Some e) -> Some e
added
removed
parsing_c/parse_c.ml
