forked from rocky/python-uncompyle6
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathparse35.py
More file actions
276 lines (233 loc) · 11.8 KB
/
parse35.py
File metadata and controls
276 lines (233 loc) · 11.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
# Copyright (c) 2016-2017, 2019 Rocky Bernstein
"""
spark grammar differences over Python 3.4 for Python 3.5.
"""
from __future__ import print_function
from uncompyle6.parser import PythonParserSingle, nop_func
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parsers.parse34 import Python34Parser
class Python35Parser(Python34Parser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python35Parser, self).__init__(debug_parser)
self.customized = {}
def p_35on(self, args):
"""
pb_ja ::= POP_BLOCK JUMP_ABSOLUTE
# The number of canned instructions in new statements is mind boggling.
# I'm sure by the time Python 4 comes around these will be turned
# into special opcodes
while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK
POP_BLOCK COME_FROM_LOOP
while1stmt ::= SETUP_LOOP l_stmts POP_BLOCK COME_FROM_LOOP
while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK
POP_BLOCK else_suite COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr returns POP_BLOCK COME_FROM_LOOP
# The following rule is for Python 3.5+ where we can have stuff like
# while ..
# if
# ...
# the end of the if will jump back to the loop and there will be a COME_FROM
# after the jump
l_stmts ::= lastl_stmt come_froms l_stmts
# Python 3.5+ Await statement
expr ::= await_expr
await_expr ::= expr GET_AWAITABLE LOAD_CONST YIELD_FROM
stmt ::= await_stmt
await_stmt ::= await_expr POP_TOP
# Python 3.5+ has WITH_CLEANUP_START/FINISH
withstmt ::= expr
SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
withasstmt ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
# Python 3.5+ async additions
stmt ::= async_for_stmt
async_for_stmt ::= SETUP_LOOP expr
GET_AITER
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK jump_except COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
JUMP_ABSOLUTE END_FINALLY COME_FROM
for_block POP_BLOCK JUMP_ABSOLUTE
COME_FROM_LOOP
async_for_stmt ::= SETUP_LOOP expr
GET_AITER
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK jump_except COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
JUMP_ABSOLUTE END_FINALLY JUMP_BACK
pass POP_BLOCK JUMP_ABSOLUTE
COME_FROM_LOOP
stmt ::= async_forelse_stmt
async_forelse_stmt ::= SETUP_LOOP expr
GET_AITER
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
JUMP_ABSOLUTE END_FINALLY COME_FROM
for_block pb_ja
else_suite COME_FROM_LOOP
inplace_op ::= INPLACE_MATRIX_MULTIPLY
binary_op ::= BINARY_MATRIX_MULTIPLY
# Python 3.5+ does jump optimization
# In <.3.5 the below is a JUMP_FORWARD to a JUMP_ABSOLUTE.
return_if_stmt ::= ret_expr RETURN_END_IF POP_BLOCK
return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM
jb_else ::= JUMP_BACK ELSE
ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
ifelsestmtl ::= testexpr c_stmts_opt jb_else else_suitel
# 3.5 Has jump optimization which can route the end of an
# "if/then" back to to a loop just before an else.
jump_absolute_else ::= jb_else
jump_absolute_else ::= CONTINUE ELSE
# Our hacky "ELSE" determination doesn't do a good job and really
# determine the start of an "else". It could also be the end of an
# "if-then" which ends in a "continue". Perhaps with real control-flow
# analysis we'll sort this out. Or call "ELSE" something more appropriate.
_ifstmts_jump ::= c_stmts_opt ELSE
# ifstmt ::= testexpr c_stmts_opt
iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD
# Python 3.3+ also has yield from. 3.5 does it
# differently than 3.3, 3.4
yield_from ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM
"""
def customize_grammar_rules(self, tokens, customize):
self.remove_rules("""
yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM
yield_from ::= expr expr YIELD_FROM
withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
withasstmt ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
""")
super(Python35Parser, self).customize_grammar_rules(tokens, customize)
for i, token in enumerate(tokens):
opname = token.kind
# FIXME: I suspect this is wrong for 3.6 and 3.5, but
# I haven't verified what the 3.7ish fix is
if opname == 'BUILD_MAP_UNPACK_WITH_CALL':
if self.version < 3.7:
self.addRule("expr ::= unmapexpr", nop_func)
nargs = token.attr % 256
map_unpack_n = "map_unpack_%s" % nargs
rule = map_unpack_n + ' ::= ' + 'expr ' * (nargs)
self.addRule(rule, nop_func)
rule = "unmapexpr ::= %s %s" % (map_unpack_n, opname)
self.addRule(rule, nop_func)
call_token = tokens[i+1]
rule = 'call ::= expr unmapexpr ' + call_token.kind
self.addRule(rule, nop_func)
elif opname == 'BEFORE_ASYNC_WITH':
# Some Python 3.5+ async additions
rules_str = """
async_with_stmt ::= expr
stmt ::= async_with_stmt
async_with_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
stmt ::= async_with_as_stmt
async_with_as_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
"""
self.addRule(rules_str, nop_func)
elif opname == 'BUILD_MAP_UNPACK':
self.addRule("""
expr ::= unmap_dict
unmap_dict ::= dict_comp BUILD_MAP_UNPACK
""", nop_func)
elif opname == 'SETUP_WITH':
# Python 3.5+ has WITH_CLEANUP_START/FINISH
rules_str = """
withstmt ::= expr
SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
withasstmt ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
"""
self.addRule(rules_str, nop_func)
pass
return
def custom_classfunc_rule(self, opname, token, customize, *args):
args_pos, args_kw = self.get_pos_kw(token)
# Additional exprs for * and ** args:
# 0 if neither
# 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
# 2 for * and ** args (CALL_FUNCTION_VAR_KW).
# Yes, this computation based on instruction name is a little bit hoaky.
nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
uniq_param = args_kw + args_pos
if frozenset(('GET_AWAITABLE', 'YIELD_FROM')).issubset(self.seen_ops):
rule = ('async_call ::= expr ' +
('pos_arg ' * args_pos) +
('kwarg ' * args_kw) +
'expr ' * nak + token.kind +
' GET_AWAITABLE LOAD_CONST YIELD_FROM')
self.add_unique_rule(rule, token.kind, uniq_param, customize)
self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize)
if opname.startswith('CALL_FUNCTION_VAR'):
# Python 3.5 changes the stack position of *args. KW args come
# after *args.
# Note: Python 3.6+ replaces CALL_FUNCTION_VAR and
# CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
token.kind = self.call_fn_name(token)
if opname.endswith('KW'):
kw = 'expr '
else:
kw = ''
rule = ('call ::= expr expr ' +
('pos_arg ' * args_pos) +
('kwarg ' * args_kw) + kw + token.kind)
# Note: semantic actions make use of the fact of wheter "args_pos"
# zero or not in creating a template rule.
self.add_unique_rule(rule, token.kind, args_pos, customize)
else:
super(Python35Parser, self).custom_classfunc_rule(opname, token, customize, *args)
class Python35ParserSingle(Python35Parser, PythonParserSingle):
pass
if __name__ == '__main__':
# Check grammar
p = Python35Parser()
p.check_grammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 3.5:
lhs, rhs, tokens, right_recursive = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
# print(sorted(p.rule2name.items()))