Skip to content

Commit dd8c216

Browse files
nevansst0012
nevans
and
st0012
authored
Drop reimplementation of Ripper lex state (#1118)
* Drop reimplementation of Ripper lex state This code was for ruby 2.4 compatibility, but rdoc dropped support for ruby 2.4 about three years ago, in f480b97. This code was almost half of the lines of code in rdoc/parser/ripper_state_lex. * Remove unused Ripper constants and const_defined? This was mostly copied from the diff in @st0012's PR comment. The remaining constants have been updated to get their value directly from Ripper. Co-authored-by: Stan Lo <[email protected]> * Use Ripper::EXPR_LABEL directly Since this is only used from outside RipperStateLex, there's no longer any benefit to using the indirect reference rather than just going straight to Ripper. --------- Co-authored-by: Stan Lo <[email protected]>
1 parent 0482993 commit dd8c216

File tree

2 files changed

+13
-311
lines changed

2 files changed

+13
-311
lines changed

‎lib/rdoc/parser/ripper_state_lex.rb‎

Lines changed: 7 additions & 305 deletions
Original file line numberDiff line numberDiff line change
@@ -7,307 +7,12 @@
77
class RDoc::Parser::RipperStateLex
88
# :stopdoc:
99

10-
# TODO: Remove this constants after Ruby 2.4 EOL
11-
RIPPER_HAS_LEX_STATE = Ripper::Filter.method_defined?(:state)
12-
1310
Token = Struct.new(:line_no, :char_no, :kind, :text, :state)
1411

15-
EXPR_NONE = 0
16-
EXPR_BEG = 1
17-
EXPR_END = 2
18-
EXPR_ENDARG = 4
19-
EXPR_ENDFN = 8
20-
EXPR_ARG = 16
21-
EXPR_CMDARG = 32
22-
EXPR_MID = 64
23-
EXPR_FNAME = 128
24-
EXPR_DOT = 256
25-
EXPR_CLASS = 512
26-
EXPR_LABEL = 1024
27-
EXPR_LABELED = 2048
28-
EXPR_FITEM = 4096
29-
EXPR_VALUE = EXPR_BEG
30-
EXPR_BEG_ANY = (EXPR_BEG | EXPR_MID | EXPR_CLASS)
31-
EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG)
32-
EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN)
33-
34-
class InnerStateLex < Ripper::Filter
35-
attr_accessor :lex_state
36-
37-
def initialize(code)
38-
@lex_state = EXPR_BEG
39-
@in_fname = false
40-
@continue = false
41-
reset
42-
super(code)
43-
end
44-
45-
def reset
46-
@command_start = false
47-
@cmd_state = @command_start
48-
end
49-
50-
def on_nl(tok, data)
51-
case @lex_state
52-
when EXPR_FNAME, EXPR_DOT
53-
@continue = true
54-
else
55-
@continue = false
56-
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
57-
end
58-
data << Token.new(lineno, column, __method__, tok, @lex_state)
59-
end
60-
61-
def on_ignored_nl(tok, data)
62-
case @lex_state
63-
when EXPR_FNAME, EXPR_DOT
64-
@continue = true
65-
else
66-
@continue = false
67-
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
68-
end
69-
data << Token.new(lineno, column, __method__, tok, @lex_state)
70-
end
71-
72-
def on_op(tok, data)
73-
case tok
74-
when '&', '|', '!', '!=', '!~'
75-
case @lex_state
76-
when EXPR_FNAME, EXPR_DOT
77-
@lex_state = EXPR_ARG
78-
else
79-
@lex_state = EXPR_BEG
80-
end
81-
when '<<'
82-
# TODO next token?
83-
case @lex_state
84-
when EXPR_FNAME, EXPR_DOT
85-
@lex_state = EXPR_ARG
86-
else
87-
@lex_state = EXPR_BEG
88-
end
89-
when '?'
90-
@lex_state = EXPR_BEG
91-
when '&&', '||', '+=', '-=', '*=', '**=',
92-
'&=', '|=', '^=', '<<=', '>>=', '||=', '&&='
93-
@lex_state = EXPR_BEG
94-
when '::'
95-
case @lex_state
96-
when EXPR_ARG, EXPR_CMDARG
97-
@lex_state = EXPR_DOT
98-
when EXPR_FNAME, EXPR_DOT
99-
@lex_state = EXPR_ARG
100-
else
101-
@lex_state = EXPR_BEG
102-
end
103-
else
104-
case @lex_state
105-
when EXPR_FNAME, EXPR_DOT
106-
@lex_state = EXPR_ARG
107-
else
108-
@lex_state = EXPR_BEG
109-
end
110-
end
111-
data << Token.new(lineno, column, __method__, tok, @lex_state)
112-
end
113-
114-
def on_kw(tok, data)
115-
case tok
116-
when 'class'
117-
@lex_state = EXPR_CLASS
118-
@in_fname = true
119-
when 'def'
120-
@lex_state = EXPR_FNAME
121-
@continue = true
122-
@in_fname = true
123-
when 'if', 'unless', 'while', 'until'
124-
if ((EXPR_MID | EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
125-
@lex_state = EXPR_BEG | EXPR_LABEL
126-
else
127-
@lex_state = EXPR_BEG
128-
end
129-
when 'begin', 'case', 'when'
130-
@lex_state = EXPR_BEG
131-
when 'return', 'break'
132-
@lex_state = EXPR_MID
133-
else
134-
if @lex_state == EXPR_FNAME
135-
@lex_state = EXPR_END
136-
else
137-
@lex_state = EXPR_END
138-
end
139-
end
140-
data << Token.new(lineno, column, __method__, tok, @lex_state)
141-
end
142-
143-
def on_tstring_beg(tok, data)
144-
@lex_state = EXPR_BEG
145-
data << Token.new(lineno, column, __method__, tok, @lex_state)
146-
end
147-
148-
def on_tstring_end(tok, data)
149-
@lex_state = EXPR_END | EXPR_ENDARG
150-
data << Token.new(lineno, column, __method__, tok, @lex_state)
151-
end
152-
153-
def on_CHAR(tok, data)
154-
@lex_state = EXPR_END
155-
data << Token.new(lineno, column, __method__, tok, @lex_state)
156-
end
157-
158-
def on_period(tok, data)
159-
@lex_state = EXPR_DOT
160-
data << Token.new(lineno, column, __method__, tok, @lex_state)
161-
end
162-
163-
def on_int(tok, data)
164-
@lex_state = EXPR_END | EXPR_ENDARG
165-
data << Token.new(lineno, column, __method__, tok, @lex_state)
166-
end
167-
168-
def on_float(tok, data)
169-
@lex_state = EXPR_END | EXPR_ENDARG
170-
data << Token.new(lineno, column, __method__, tok, @lex_state)
171-
end
172-
173-
def on_rational(tok, data)
174-
@lex_state = EXPR_END | EXPR_ENDARG
175-
data << Token.new(lineno, column, __method__, tok, @lex_state)
176-
end
177-
178-
def on_imaginary(tok, data)
179-
@lex_state = EXPR_END | EXPR_ENDARG
180-
data << Token.new(lineno, column, __method__, tok, @lex_state)
181-
end
182-
183-
def on_symbeg(tok, data)
184-
@lex_state = EXPR_FNAME
185-
@continue = true
186-
@in_fname = true
187-
data << Token.new(lineno, column, __method__, tok, @lex_state)
188-
end
189-
190-
private def on_variables(event, tok, data)
191-
if @in_fname
192-
@lex_state = EXPR_ENDFN
193-
@in_fname = false
194-
@continue = false
195-
elsif @continue
196-
case @lex_state
197-
when EXPR_DOT
198-
@lex_state = EXPR_ARG
199-
else
200-
@lex_state = EXPR_ENDFN
201-
@continue = false
202-
end
203-
else
204-
@lex_state = EXPR_CMDARG
205-
end
206-
data << Token.new(lineno, column, event, tok, @lex_state)
207-
end
208-
209-
def on_ident(tok, data)
210-
on_variables(__method__, tok, data)
211-
end
212-
213-
def on_ivar(tok, data)
214-
@lex_state = EXPR_END
215-
on_variables(__method__, tok, data)
216-
end
217-
218-
def on_cvar(tok, data)
219-
@lex_state = EXPR_END
220-
on_variables(__method__, tok, data)
221-
end
222-
223-
def on_gvar(tok, data)
224-
@lex_state = EXPR_END
225-
on_variables(__method__, tok, data)
226-
end
227-
228-
def on_backref(tok, data)
229-
@lex_state = EXPR_END
230-
on_variables(__method__, tok, data)
231-
end
232-
233-
def on_lparen(tok, data)
234-
@lex_state = EXPR_LABEL | EXPR_BEG
235-
data << Token.new(lineno, column, __method__, tok, @lex_state)
236-
end
237-
238-
def on_rparen(tok, data)
239-
@lex_state = EXPR_ENDFN
240-
data << Token.new(lineno, column, __method__, tok, @lex_state)
241-
end
242-
243-
def on_lbrace(tok, data)
244-
@lex_state = EXPR_LABEL | EXPR_BEG
245-
data << Token.new(lineno, column, __method__, tok, @lex_state)
246-
end
247-
248-
def on_rbrace(tok, data)
249-
@lex_state = EXPR_ENDARG
250-
data << Token.new(lineno, column, __method__, tok, @lex_state)
251-
end
252-
253-
def on_lbracket(tok, data)
254-
@lex_state = EXPR_LABEL | EXPR_BEG
255-
data << Token.new(lineno, column, __method__, tok, @lex_state)
256-
end
257-
258-
def on_rbracket(tok, data)
259-
@lex_state = EXPR_ENDARG
260-
data << Token.new(lineno, column, __method__, tok, @lex_state)
261-
end
262-
263-
def on_const(tok, data)
264-
case @lex_state
265-
when EXPR_FNAME
266-
@lex_state = EXPR_ENDFN
267-
when EXPR_CLASS, EXPR_CMDARG, EXPR_MID
268-
@lex_state = EXPR_ARG
269-
else
270-
@lex_state = EXPR_CMDARG
271-
end
272-
data << Token.new(lineno, column, __method__, tok, @lex_state)
273-
end
274-
275-
def on_sp(tok, data)
276-
data << Token.new(lineno, column, __method__, tok, @lex_state)
277-
end
278-
279-
def on_comma(tok, data)
280-
@lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
281-
data << Token.new(lineno, column, __method__, tok, @lex_state)
282-
end
283-
284-
def on_comment(tok, data)
285-
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
286-
data << Token.new(lineno, column, __method__, tok, @lex_state)
287-
end
288-
289-
def on_ignored_sp(tok, data)
290-
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
291-
data << Token.new(lineno, column, __method__, tok, @lex_state)
292-
end
293-
294-
def on_heredoc_beg(tok, data)
295-
data << Token.new(lineno, column, __method__, tok, @lex_state)
296-
@lex_state = EXPR_END
297-
data
298-
end
299-
300-
def on_heredoc_end(tok, data)
301-
data << Token.new(lineno, column, __method__, tok, @lex_state)
302-
@lex_state = EXPR_BEG
303-
data
304-
end
305-
306-
def on_default(event, tok, data)
307-
reset
308-
data << Token.new(lineno, column, event, tok, @lex_state)
309-
end
310-
end unless RIPPER_HAS_LEX_STATE
12+
EXPR_END = Ripper::EXPR_END
13+
EXPR_ENDFN = Ripper::EXPR_ENDFN
14+
EXPR_ARG = Ripper::EXPR_ARG
15+
EXPR_FNAME = Ripper::EXPR_FNAME
31116

31217
class InnerStateLex < Ripper::Filter
31318
def initialize(code)
@@ -317,7 +22,7 @@ def initialize(code)
31722
def on_default(event, tok, data)
31823
data << Token.new(lineno, column, event, tok, state)
31924
end
320-
end if RIPPER_HAS_LEX_STATE
25+
end
32126

32227
def get_squashed_tk
32328
if @buf.empty?
@@ -333,9 +38,8 @@ def get_squashed_tk
33338
tk = get_string_tk(tk)
33439
when :on_backtick then
33540
if (tk[:state] & (EXPR_FNAME | EXPR_ENDFN)) != 0
336-
@inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
33741
tk[:kind] = :on_ident
338-
tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
42+
tk[:state] = Ripper::Lexer::State.new(EXPR_ARG)
33943
else
34044
tk = get_string_tk(tk)
34145
end
@@ -345,7 +49,6 @@ def get_squashed_tk
34549
tk = get_embdoc_tk(tk)
34650
when :on_heredoc_beg then
34751
@heredoc_queue << retrieve_heredoc_info(tk)
348-
@inner_lex.lex_state = EXPR_END unless RIPPER_HAS_LEX_STATE
34952
when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
35053
if !@heredoc_queue.empty?
35154
get_heredoc_tk(*@heredoc_queue.shift)
@@ -549,8 +252,7 @@ def get_squashed_tk
549252
private def get_op_tk(tk)
550253
redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~]
551254
if redefinable_operators.include?(tk[:text]) and tk[:state] == EXPR_ARG then
552-
@inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
553-
tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
255+
tk[:state] = Ripper::Lexer::State.new(EXPR_ARG)
554256
tk[:kind] = :on_ident
555257
elsif tk[:text] =~ /^[-+]$/ then
556258
tk_ahead = get_squashed_tk

0 commit comments

Comments
 (0)