107
115
when :BULLET, :LABEL, :LALPHA, :NOTE, :NUMBER, :UALPHA then
110
if column < margin then
115
if list.type and list.type != list_type then
120
list.type = list_type
117
if column < margin || (list.type && list.type != type) then
123
peek_type, _, column, = peek_token
123
126
when :NOTE, :LABEL then
124
_, indent, = get # SPACE
125
if :NEWLINE == peek_token.first then
127
peek_type, new_indent, peek_column, = peek_token
128
indent = new_indent if
129
peek_type == :INDENT and peek_column >= column
127
if peek_type == :NEWLINE then
128
# description not on the same line as LABEL/NOTE
129
# skip the trailing newline & any blank lines below
130
while peek_type == :NEWLINE
132
peek_type, _, column, = peek_token
137
# - at a column < margin:
140
# - at the same column, but with a different type of list item
143
# - at the same column, with the same type of list item
146
# In all cases, we have an empty description.
147
# In the last case only, we continue.
148
if peek_type.nil? || column < margin then
150
elsif column == margin then
164
item = RDoc::Markup::ListItem.new(data)
165
item << RDoc::Markup::BlankLine.new
137
list_item = build_list_item(margin + indent, data)
175
list_item = RDoc::Markup::ListItem.new data
176
parse list_item, column
139
list << list_item if list_item
154
# Builds a ListItem that is flush to +indent+ with type +item_type+
156
def build_list_item indent, item_type = nil
157
p :list_item_start => [indent, item_type] if @debug
159
list_item = RDoc::Markup::ListItem.new item_type
161
until @tokens.empty? do
162
type, data, column = get
164
if column < indent and
165
not type == :NEWLINE and
166
(type != :INDENT or data < indent) then
174
list_item.push(*parse(indent))
177
list_item << build_paragraph(indent)
179
list_item << build_heading(data)
181
list_item << RDoc::Markup::BlankLine.new
182
when *LIST_TOKENS then
184
list_item << build_list(column)
186
raise ParseError, "Unhandled token #{@current_token.inspect}"
190
p :list_item_end => [indent, item_type] if @debug
192
return nil if list_item.empty?
194
list_item.parts.shift if
195
RDoc::Markup::BlankLine === list_item.parts.first and
202
193
# Builds a Paragraph that is flush to +margin+
204
195
def build_paragraph margin
238
# Builds a Verbatim that is flush to +margin+
218
# Builds a Verbatim that is indented from +margin+.
220
# The verbatim block is shifted left (the least indented lines start in
221
# column 0). Each part of the verbatim is one line of text, always
222
# terminated by a newline. Blank lines always consist of a single newline
223
# character, and there is never a single newline at the end of the verbatim.
240
225
def build_verbatim margin
241
226
p :verbatim_begin => margin if @debug
242
227
verbatim = RDoc::Markup::Verbatim.new
230
generate_leading_spaces = true
244
233
until @tokens.empty? do
245
234
type, data, column, = get
236
if type == :NEWLINE then
240
generate_leading_spaces = true
249
if generate_leading_spaces then
250
indent = column - margin
252
min_indent = indent if min_indent.nil? || indent < min_indent
253
generate_leading_spaces = false
249
if margin >= data then
254
indent = data - margin
256
verbatim << ' ' * indent
257
257
when :HEADER then
258
verbatim << '=' * data
260
259
_, _, peek_column, = peek_token
261
260
peek_column ||= column + data
262
verbatim << ' ' * (peek_column - column - data)
261
indent = peek_column - column - data
265
verbatim << '-' * width
267
266
_, _, peek_column, = peek_token
268
peek_column ||= column + data + 2
269
verbatim << ' ' * (peek_column - column - width)
267
peek_column ||= column + width
268
indent = peek_column - column - width
272
when *LIST_TOKENS then
273
if column <= margin then
278
273
list_marker = case type
279
when :BULLET then '*'
280
when :LABEL then "[#{data}]"
281
when :LALPHA, :NUMBER, :UALPHA then "#{data}."
282
when :NOTE then "#{data}::"
274
when :BULLET then data
275
when :LABEL then "[#{data}]"
276
when :NOTE then "#{data}::"
277
else # :LALPHA, :NUMBER, :UALPHA
285
verbatim << list_marker
289
verbatim << ' ' * (data - list_marker.length)
292
break unless [:INDENT, :NEWLINE].include? peek_token[0]
281
peek_type, _, peek_column = peek_token
282
unless peek_type == :NEWLINE then
283
peek_column ||= column + list_marker.length
284
indent = peek_column - column - list_marker.length
291
verbatim << line << "\n" unless line.empty?
292
verbatim.parts.each { |p| p.slice!(0, min_indent) unless p == "\n" } if min_indent > 0
299
293
verbatim.normalize
301
295
p :verbatim_end => margin if @debug
316
# Parses the tokens into a Document
310
# Parses the tokens into an array of RDoc::Markup::XXX objects,
311
# and appends them to the passed +parent+ RDoc::Markup::YYY object.
313
# Exits at the end of the token stream, or when it encounters a token
314
# in a column less than +indent+ (unless it is a NEWLINE).
318
def parse parent, indent = 0
319
319
p :parse_start => indent if @debug
323
321
until @tokens.empty? do
324
322
type, data, column, = get
326
if type != :INDENT and column < indent then
333
document << build_heading(data)
335
if indent > data then
338
elsif indent == data then
343
document << build_verbatim(indent)
345
document << RDoc::Markup::BlankLine.new
324
if type == :NEWLINE then
325
# trailing newlines are skipped below, so this is a blank line
326
parent << RDoc::Markup::BlankLine.new
346
327
skip :NEWLINE, false
331
# indentation change: break or verbatim
332
if column < indent then
335
elsif column > indent then
337
parent << build_verbatim(indent)
341
# indentation is the same
344
parent << build_heading(data)
348
document << RDoc::Markup::Rule.new(data)
346
parent << RDoc::Markup::Rule.new(data)
352
document << build_paragraph(indent)
354
# we're done with this paragraph (indent mismatch)
355
break if peek_token[0] == :TEXT
350
parent << build_paragraph(indent)
356
351
when *LIST_TOKENS then
359
list = build_list(indent)
361
document << list if list
363
# we're done with this list (indent mismatch)
364
break if LIST_TOKENS.include? peek_token.first and indent > 0
353
parent << build_list(indent)
366
355
type, data, column, line = @current_token
368
"Unhandled token #{type} (#{data.inspect}) at #{line}:#{column}"
356
raise ParseError, "Unhandled token #{type} (#{data.inspect}) at #{line}:#{column}"
372
360
p :parse_end => indent if @debug
387
# Skips a token of +token_type+, optionally raising an error.
376
# Skips the next token if its type is +token_type+.
378
# Optionally raises an error if the next token is not of the expected type.
389
380
def skip token_type, error = true
392
382
return unless type # end of stream
394
383
return @current_token if token_type == type
398
raise ParseError, "expected #{token_type} got #{@current_token.inspect}" if
403
# Consumes tokens until NEWLINE and turns them back into text
413
_, space, = get # SPACE
414
"*#{' ' * (space - 1)}"
416
_, space, = get # SPACE
417
"[#{data}]#{' ' * (space - data.length - 2)}"
418
when :LALPHA, :NUMBER, :UALPHA then
419
_, space, = get # SPACE
420
"#{data}.#{' ' * (space - 2)}"
422
_, space = get # SPACE
423
"#{data}::#{' ' * (space - data.length - 2)}"
432
raise ParseError, "unhandled token #{@current_token.inspect}"
440
# Calculates the column and line of the current token based on +offset+.
443
[offset - @line_pos, @line]
385
raise ParseError, "expected #{token_type} got #{@current_token.inspect}" if error
400
# leading spaces will be reflected by the column of the next token
401
# the only thing we loose are trailing spaces at the end of the file
404
# note: after BULLET, LABEL, etc.,
405
# indent will be the column of the next non-newline token
459
409
when s.scan(/\r?\n/) then
460
410
token = [:NEWLINE, s.matched, *token_pos(pos)]
461
411
@line_pos = s.pos
464
when s.scan(/ +/) then
465
[:INDENT, s.matched_size, *token_pos(pos)]
466
when s.scan(/(=+)\s*/) then
414
# === text => :HEADER then :TEXT
415
when s.scan(/(=+)(\s*)/) then
467
416
level = s[1].length
468
level = 6 if level > 6
469
@tokens << [:HEADER, level, *token_pos(pos)]
417
header = [:HEADER, level, *token_pos(pos)]
473
[:TEXT, s.matched, *token_pos(pos)]
474
when s.scan(/^(-{3,}) *$/) then
419
if s[2] =~ /^\r?\n/ then
426
[:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
428
# --- (at least 3) and nothing else on the line => :RULE
429
when s.scan(/(-{3,}) *$/) then
475
430
[:RULE, s[1].length - 2, *token_pos(pos)]
476
when s.scan(/([*-])\s+/) then
477
@tokens << [:BULLET, :BULLET, *token_pos(pos)]
478
[:SPACE, s.matched_size, *token_pos(pos)]
479
when s.scan(/([a-z]|\d+)\.[ \t]+\S/i) then
431
# * or - followed by white space and text => :BULLET
432
when s.scan(/([*-]) +(\S)/) then
433
s.pos -= s[2].bytesize # unget \S
434
[:BULLET, s[1], *token_pos(pos)]
435
# A. text, a. text, 12. text => :UALPHA, :LALPHA, :NUMBER
436
when s.scan(/([a-z]|\d+)\. +(\S)/i) then
437
# FIXME if tab(s), the column will be wrong
438
# either support tabs everywhere by first expanding them to
439
# spaces, or assume that they will have been replaced
440
# before (and provide a check for that at least in debug
480
442
list_label = s[1]
481
width = s.matched_size - 1
483
s.pos -= 1 # unget \S
485
list_type = case list_label
486
when /[a-z]/ then :LALPHA
487
when /[A-Z]/ then :UALPHA
488
when /\d/ then :NUMBER
490
raise ParseError, "BUG token #{list_label}"
493
@tokens << [list_type, list_label, *token_pos(pos)]
494
[:SPACE, width, *token_pos(pos)]
443
s.pos -= s[2].bytesize # unget \S
446
when /[a-z]/ then :LALPHA
447
when /[A-Z]/ then :UALPHA
448
when /\d/ then :NUMBER
450
raise ParseError, "BUG token #{list_label}"
452
[list_type, list_label, *token_pos(pos)]
453
# [text] followed by spaces or end of line => :LABEL
495
454
when s.scan(/\[(.*?)\]( +|$)/) then
496
@tokens << [:LABEL, s[1], *token_pos(pos)]
497
[:SPACE, s.matched_size, *token_pos(pos)]
455
[:LABEL, s[1], *token_pos(pos)]
456
# text:: followed by spaces or end of line => :NOTE
498
457
when s.scan(/(.*?)::( +|$)/) then
499
@tokens << [:NOTE, s[1], *token_pos(pos)]
500
[:SPACE, s.matched_size, *token_pos(pos)]
458
[:NOTE, s[1], *token_pos(pos)]
459
# anything else: :TEXT
501
460
else s.scan(/.*/)
502
[:TEXT, s.matched, *token_pos(pos)]
461
[:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
added
removed
lib/rdoc/markup/parser.rb
