forked from rocky/python-uncompyle6
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathparse26.py
More file actions
283 lines (221 loc) · 11.4 KB
/
parse26.py
File metadata and controls
283 lines (221 loc) · 11.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
# Copyright (c) 2016 Rocky Bernstein
"""
spark grammar differences over Python2 for Python 2.6.
"""
from uncompyle6.parser import PythonParserSingle
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parsers.parse2 import Python2Parser
class Python26Parser(Python2Parser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python26Parser, self).__init__(debug_parser)
self.customized = {}
def p_try_except26(self, args):
"""
except_stmt ::= except_cond3 except_suite
except_cond1 ::= DUP_TOP expr COMPARE_OP
JUMP_IF_FALSE POP_TOP POP_TOP POP_TOP POP_TOP
except_cond3 ::= DUP_TOP expr COMPARE_OP
JUMP_IF_FALSE POP_TOP POP_TOP designator POP_TOP
try_middle ::= JUMP_FORWARD COME_FROM except_stmts
come_from_pop END_FINALLY COME_FROM
try_middle ::= jmp_abs COME_FROM except_stmts
POP_TOP END_FINALLY
try_middle ::= jmp_abs COME_FROM except_stmts
END_FINALLY JUMP_FORWARD
# Sometimes we don't put in COME_FROM to the next statement
# like we do in 2.7. Perhaps we should?
trystmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
try_middle
tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
try_middle else_suite COME_FROM
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
except_suite ::= c_stmts_opt JUMP_FORWARD come_from_pop
except_suite ::= c_stmts_opt JUMP_FORWARD POP_TOP
except_suite ::= c_stmts_opt jmp_abs come_from_pop
# Python 3 also has this.
come_froms ::= come_froms COME_FROM
come_froms ::= COME_FROM
# This is what happens after a jump where
# we start a new block. For reasons I don't fully
# understand, there is also a value on the top of the stack
come_from_pop ::= COME_FROM POP_TOP
come_froms_pop ::= come_froms POP_TOP
"""
# In contrast to Python 2.7, Python 2.6 has a lot of
# POP_TOP's which come right after various jumps.
# The COME_FROM instructions our scanner adds, here it is to assist
# distinguishing the extraneous POP_TOPs from those that start
# after one of these jumps
def p_jumps26(self, args):
"""
# The are the equivalents of Python 2.7+'s
# POP_JUMP_IF_TRUE and POP_JUMP_IF_FALSE
jmp_true ::= JUMP_IF_TRUE POP_TOP
jmp_false ::= JUMP_IF_FALSE POP_TOP
jf_pop ::= JUMP_FORWARD POP_TOP
jf_pop ::= JUMP_ABSOLUTE POP_TOP
jb_pop ::= JUMP_BACK POP_TOP
jb_cont ::= JUMP_BACK
jb_cont ::= CONTINUE
jb_cf_pop ::= JUMP_BACK come_froms POP_TOP
jb_cf_pop ::= JUMP_BACK POP_TOP
ja_cf_pop ::= JUMP_ABSOLUTE come_froms POP_TOP
jf_cf_pop ::= JUMP_FORWARD come_froms POP_TOP
bp_come_from ::= POP_BLOCK COME_FROM
jb_bp_come_from ::= JUMP_BACK bp_come_from
_ifstmts_jump ::= c_stmts_opt jf_pop COME_FROM
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD come_froms POP_TOP COME_FROM
# This is what happens after a jump where
# we start a new block. For reasons I don't fully
# understand, there is also a value on the top of the stack
come_froms_pop ::= come_froms POP_TOP
"""
def p_stmt26(self, args):
"""
# We use filler as a placeholder to keep nonterminal positions
# the same across different grammars so that the same semantic actions
# can be used
filler ::=
assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 come_froms_pop
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_2 come_froms_pop
break_stmt ::= BREAK_LOOP JUMP_BACK
# Semantic actions want else_suitel to be at index 3
ifelsestmtl ::= testexpr c_stmts_opt jb_cf_pop else_suitel
ifelsestmtc ::= testexpr c_stmts_opt ja_cf_pop else_suitec
# Semantic actions want suite_stmts_opt to be at index 3
withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
# Semantic actions want designator to be at index 2
# Rule is possibly 2.6 only
withasstmt ::= expr setupwithas designator suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
# This is truly weird. 2.7 does this (not including POP_TOP) with
# opcode SETUP_WITH
setupwith ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP
# Possibly 2.6 only
setupwithas ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 setup_finally
setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
setup_finally ::= STORE_NAME SETUP_FINALLY LOAD_NAME DELETE_NAME
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_pop POP_BLOCK _come_from
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_cf_pop bp_come_from
whilestmt ::= SETUP_LOOP testexpr return_stmts come_froms POP_TOP bp_come_from
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_pop POP_BLOCK
else_suite COME_FROM
return_stmt ::= ret_expr RETURN_END_IF POP_TOP
return_stmt ::= ret_expr RETURN_VALUE POP_TOP
return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK come_from_pop
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_from_pop
while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
ifstmt ::= testexpr_then _ifstmts_jump
# Semantic actions want the else to be at position 3
ifelsestmt ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
ifelsestmt ::= testexpr_then c_stmts_opt jf_cf_pop else_suite come_froms
ifelsestmt ::= testexpr c_stmts_opt filler else_suitel come_froms POP_TOP
ifelsestmt ::= testexpr_then c_stmts_opt filler else_suitel come_froms POP_TOP
# Semantic actions want else_suitel to be at index 3
ifelsestmtl ::= testexpr_then c_stmts_opt jb_cf_pop else_suitel
ifelsestmtc ::= testexpr_then c_stmts_opt ja_cf_pop else_suitec
iflaststmt ::= testexpr_then c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
testexpr_then ::= testtrue_then
testexpr_then ::= testfalse_then
testtrue_then ::= expr jmp_true_then
testfalse_then ::= expr jmp_false_then
jmp_false_then ::= JUMP_IF_FALSE THEN POP_TOP
jmp_true_then ::= JUMP_IF_TRUE THEN POP_TOP
# Common with 2.7
while1stmt ::= SETUP_LOOP return_stmts bp_come_from
while1stmt ::= SETUP_LOOP return_stmts COME_FROM
"""
def p_comp26(self, args):
'''
list_for ::= expr _for designator list_iter JUMP_BACK come_froms POP_TOP
# The JUMP FORWARD below jumps to the JUMP BACK. It seems to happen
# in rare cases that may have to with length of code
list_for ::= expr _for designator list_iter JUMP_FORWARD come_froms POP_TOP
COME_FROM JUMP_BACK
list_for ::= expr _for designator list_iter jb_cont
list_iter ::= list_if JUMP_BACK
list_iter ::= list_if JUMP_BACK COME_FROM POP_TOP
list_compr ::= BUILD_LIST_0 DUP_TOP
designator list_iter del_stmt
list_compr ::= BUILD_LIST_0 DUP_TOP
designator list_iter JUMP_BACK del_stmt
lc_body ::= LOAD_NAME expr LIST_APPEND
lc_body ::= LOAD_FAST expr LIST_APPEND
comp_for ::= SETUP_LOOP expr _for designator comp_iter jb_bp_come_from
comp_body ::= gen_comp_body
for_block ::= l_stmts_opt _come_from POP_TOP JUMP_BACK
# Make sure we keep indices the same as 2.7
setup_loop_lf ::= SETUP_LOOP LOAD_FAST
genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter jb_bp_come_from
genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter JUMP_BACK come_from_pop
jb_bp_come_from
genexpr ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM
list_if ::= list_if ::= expr jmp_false_then list_iter
'''
def p_ret26(self, args):
'''
ret_and ::= expr jmp_false ret_expr_or_cond COME_FROM
ret_or ::= expr jmp_true ret_expr_or_cond COME_FROM
ret_cond ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
ret_cond ::= expr jmp_false_then expr ret_expr_or_cond
ret_cond_not ::= expr jmp_true_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
return_stmt ::= ret_expr RETURN_VALUE POP_TOP
# FIXME: split into Python 2.5
ret_or ::= expr jmp_true ret_expr_or_cond come_froms
'''
def p_except26(self, args):
"""
except_suite ::= c_stmts_opt jmp_abs POP_TOP
"""
def p_misc26(self, args):
"""
conditional ::= expr jmp_false expr jf_cf_pop expr come_from_opt
and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
conditional_lambda ::= expr jmp_false_then return_if_stmt return_stmt LAMBDA_MARKER
"""
def add_custom_rules(self, tokens, customize):
super(Python26Parser, self).add_custom_rules(tokens, customize)
self.check_reduce['and'] = 'AST'
def reduce_is_invalid(self, rule, ast, tokens, first, last):
invalid = super(Python26Parser,
self).reduce_is_invalid(rule, ast,
tokens, first, last)
if invalid:
return invalid
if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
# Test that jmp_false jumps to the end of "and"
# or that it jumps to the same place as the end of "and"
jmp_false = ast[1][0]
jmp_target = jmp_false.offset + jmp_false.attr + 3
return not (jmp_target == tokens[last].offset or
tokens[last].pattr == jmp_false.pattr)
return False
class Python26ParserSingle(Python2Parser, PythonParserSingle):
pass
if __name__ == '__main__':
# Check grammar
p = Python26Parser()
p.checkGrammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 2.6:
lhs, rhs, tokens, right_recursive = p.checkSets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
LAMBDA_MARKER RETURN_LAST
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
# print(sorted(p.rule2name.items()))