def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):

self.added_rules = set()

super(Python3Parser, self).__init__(SyntaxTree, 'stmts', debug=debug_parser)

self.new_rules = set()

def p_comprehension3(self, args):

"""

def p_dict_comp3(self, args):

""""

def p_grammar(self, args):

'''

def p_misc3(self, args):

"""

def p_def_annotations3(self, args):

"""

def p_come_from3(self, args):

"""

def p_jump3(self, args):

"""

def p_stmt3(self, args):

"""

def p_loop_stmt3(self, args):

"""

def p_generator_exp3(self, args):

'''

def p_expr3(self, args):

"""

@staticmethod

def call_fn_name(token):

"""Customize CALL_FUNCTION to add the number of positional arguments"""

if token.attr is not None:

return '%s_%i' % (token.kind, token.attr)

else:

return '%s_0' % (token.kind)

def custom_build_class_rule(self, opname, i, token, tokens, customize):

'''

# FIXME: I bet this can be simplified

# look for next MAKE_FUNCTION

for i in range(i+1, len(tokens)):

if tokens[i].kind.startswith('MAKE_FUNCTION'):

break

elif tokens[i].kind.startswith('MAKE_CLOSURE'):

break

pass

assert i < len(tokens), "build_class needs to find MAKE_FUNCTION or MAKE_CLOSURE"

assert tokens[i+1].kind == 'LOAD_CONST', \

"build_class expecting CONST after MAKE_FUNCTION/MAKE_CLOSURE"

call_fn_tok = None

for i in range(i, len(tokens)):

if tokens[i].kind.startswith('CALL_FUNCTION'):

call_fn_tok = tokens[i]

break

if not call_fn_tok:

raise RuntimeError("build_class custom rule for %s needs to find CALL_FUNCTION"

% opname)

# customize build_class rule

# FIXME: What's the deal with the two rules? Different Python versions?

# Different situations? Note that the above rule is based on the CALL_FUNCTION

# token found, while this one doesn't.

if self.version < 3.6:

call_function = self.call_fn_name(call_fn_tok)

args_pos, args_kw = self.get_pos_kw(call_fn_tok)

rule = ("build_class ::= LOAD_BUILD_CLASS mkfunc %s"

"%s" % (('expr ' * (args_pos - 1) + ('kwarg ' * args_kw)),

call_function))

else:

# 3.6+ handling

call_function = call_fn_tok.kind

if call_function.startswith("CALL_FUNCTION_KW"):

self.addRule("classdef ::= build_class_kw store", nop_func)

rule = ("build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s"

% ('expr ' * (call_fn_tok.attr - 1), call_function))

else:

call_function = self.call_fn_name(call_fn_tok)

rule = ("build_class ::= LOAD_BUILD_CLASS mkfunc %s%s"

% ('expr ' * (call_fn_tok.attr - 1), call_function))

self.addRule(rule, nop_func)

return

def custom_classfunc_rule(self, opname, token, customize, next_token):

"""

args_pos, args_kw = self.get_pos_kw(token)

# Additional exprs for * and ** args:

# 0 if neither

# 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW

# 2 for * and ** args (CALL_FUNCTION_VAR_KW).

# Yes, this computation based on instruction name is a little bit hoaky.

nak = ( len(opname)-len('CALL_FUNCTION') ) // 3

token.kind = self.call_fn_name(token)

uniq_param = args_kw + args_pos

# Note: 3.5+ have subclassed this method; so we don't handle

# 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here.

rule = ('call ::= expr ' +

('pos_arg ' * args_pos) +

('kwarg ' * args_kw) +

'expr ' * nak + token.kind)

self.add_unique_rule(rule, token.kind, uniq_param, customize)

if 'LOAD_BUILD_CLASS' in self.seen_ops:

if (next_token == 'CALL_FUNCTION' and next_token.attr == 1

and args_pos > 1):

rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'

% (('expr ' * (args_pos-1)), opname, args_pos))

self.add_unique_rule(rule, token.kind, uniq_param, customize)

def add_make_function_rule(self, rule, opname, attr, customize):

"""Python 3.3 added a an addtional LOAD_CONST before MAKE_FUNCTION and

if self.version >= 3.3:

new_rule = rule % (('LOAD_CONST ') * 1)

else:

new_rule = rule % (('LOAD_CONST ') * 0)

self.add_unique_rule(new_rule, opname, attr, customize)

def customize_grammar_rules(self, tokens, customize):

"""The base grammar we start out for a Python version even with the

is_pypy = False

# For a rough break out on the first word. This may

# include instructions that don't need customization,

# but we'll do a finer check after the rough breakout.

customize_instruction_basenames = frozenset(

('BUILD', 'CALL', 'CONTINUE', 'DELETE', 'GET',

'JUMP', 'LOAD', 'LOOKUP', 'MAKE',

'RETURN', 'RAISE', 'SETUP',

'UNPACK'))

# Opcode names in the custom_ops_processed set have rules that get added

# unconditionally and the rules are constant. So they need to be done

# only once and if we see the opcode a second we don't have to consider

# adding more rules.

#

# Note: BUILD_TUPLE_UNPACK_WITH_CALL gets considered by

# default because it starts with BUILD. So we'll set to ignore it from

# the start.

custom_ops_processed = set(('BUILD_TUPLE_UNPACK_WITH_CALL',))

# A set of instruction operation names that exist in the token stream.

# We use this customize the grammar that we create.

# 2.6-compatible set comprehensions

self.seen_ops = frozenset([t.kind for t in tokens])

self.seen_op_basenames = frozenset([opname[:opname.rfind('_')] for opname in self.seen_ops])

# Loop over instructions adding custom grammar rules based on

# a specific instruction seen.

if 'PyPy' in customize:

is_pypy = True

self.addRule("""

""", nop_func)

n = len(tokens)

# Determine if we have an iteration CALL_FUNCTION_1.

has_get_iter_call_function1 = False

max_branches = 0

for i, token in enumerate(tokens):

if token == 'GET_ITER' and i < n-2 and self.call_fn_name(tokens[i+1]) == 'CALL_FUNCTION_1':

has_get_iter_call_function1 = True

max_branches += 1

elif (token == 'GET_AWAITABLE' and i < n-3

and tokens[i+1] == 'LOAD_CONST' and tokens[i+2] == 'YIELD_FROM'):

max_branches += 1

if max_branches > 2:

break

for i, token in enumerate(tokens):

opname = token.kind

# Do a quick breakout before testing potentially

# each of the dozen or so instruction in if elif.

if (opname[:opname.find('_')] not in customize_instruction_basenames

or opname in custom_ops_processed):

continue

opname_base = opname[:opname.rfind('_')]

# The order of opname listed is roughly sorted below

if opname_base == 'BUILD_CONST_KEY_MAP':

# This is in 3.6+

kvlist_n = 'expr ' * (token.attr)

rule = "dict ::= %sLOAD_CONST %s" % (kvlist_n, opname)

self.addRule(rule, nop_func)

elif opname.startswith('BUILD_LIST_UNPACK'):

v = token.attr

rule = 'build_list_unpack ::= %s%s' % ('expr ' * v, opname)

self.addRule(rule, nop_func)

rule = 'expr ::= build_list_unpack'

self.addRule(rule, nop_func)

elif opname_base in ('BUILD_MAP', 'BUILD_MAP_UNPACK'):

kvlist_n = "kvlist_%s" % token.attr

if opname == 'BUILD_MAP_n':

# PyPy sometimes has no count. Sigh.

rule = ('dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store '

'comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST')

self.add_unique_rule(rule, 'dict_comp_func', 1, customize)

kvlist_n = 'kvlist_n'

rule = 'kvlist_n ::= kvlist_n kv3'

self.add_unique_rule(rule, 'kvlist_n', 0, customize)

rule = 'kvlist_n ::='

self.add_unique_rule(rule, 'kvlist_n', 1, customize)

rule = "dict ::= BUILD_MAP_n kvlist_n"

elif self.version >= 3.5:

if not opname.startswith('BUILD_MAP_WITH_CALL'):

# FIXME: Use the attr

# so this doesn't run into exponential parsing time.

if opname.startswith('BUILD_MAP_UNPACK'):

self.add_unique_rule(rule, opname, token.attr, customize)

rule = 'dict_entry ::= ' + 'expr ' * (token.attr*2)

self.add_unique_rule(rule, opname, token.attr, customize)

# FIXME: start here. The LHS should be unmap_dict, not dict.

# FIXME: really we need a combination of dict_entry-like things.

# It just so happens the most common case is not to mix

# dictionary comphensions with dictionary, elements

if 'LOAD_DICTCOMP' in self.seen_ops:

rule = 'dict ::= %s%s' % ('dict_comp ' * token.attr, opname)

self.addRule(rule, nop_func)

rule = """

""" % ('expr ' * token.attr, opname)

else:

rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)

self.add_unique_rule(rule, opname, token.attr, customize)

rule = "dict ::= %s" % kvlist_n

else:

rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr

self.add_unique_rule(rule, opname, token.attr, customize)

rule = "dict ::= %s %s" % (opname, kvlist_n)

self.add_unique_rule(rule, opname, token.attr, customize)

elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'):

v = token.attr

rule = 'build_map_unpack_with_call ::= %s%s' % ('expr ' * v, opname)

self.addRule(rule, nop_func)

elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):

v = token.attr

rule = ('starred ::= %s %s' % ('expr ' * v, opname))

self.addRule(rule, nop_func)

elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE',

'BUILD_TUPLE_UNPACK'):

v = token.attr

is_LOAD_CLOSURE = False

if opname_base == 'BUILD_TUPLE':

# If is part of a "load_closure", then it is not part of a

# "list".

is_LOAD_CLOSURE = True

for j in range(v):

if tokens[i-j-1].kind != 'LOAD_CLOSURE':

is_LOAD_CLOSURE = False

break

if is_LOAD_CLOSURE:

rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))

self.add_unique_rule(rule, opname, token.attr, customize)

if not is_LOAD_CLOSURE or v == 0:

# We do this complicated test to speed up parsing of

# pathelogically long literals, especially those over 1024.

build_count = token.attr

thousands = (build_count//1024)

thirty32s = ((build_count//32) % 32)

if thirty32s > 0:

rule = "expr32 ::=%s" % (' expr' * 32)

self.add_unique_rule(rule, opname_base, build_count, customize)

pass

if thousands > 0:

self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32),

opname_base, build_count, customize)

pass

collection = opname_base[opname_base.find('_')+1:].lower()

rule = (('%s ::= ' % collection) + 'expr1024 '*thousands +

'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname)

self.add_unique_rules([

"expr ::= %s" % collection,

rule], customize)

continue

continue

elif opname_base == 'BUILD_SLICE':

if token.attr == 2:

self.add_unique_rules([

'expr ::= build_slice2',

'build_slice2 ::= expr expr BUILD_SLICE_2'

], customize)

else:

assert token.attr == 3, "BUILD_SLICE value must be 2 or 3; is %s" % v

self.add_unique_rules([

'expr ::= build_slice3',

'build_slice3 ::= expr expr expr BUILD_SLICE_3',

], customize)

elif (opname in frozenset(('CALL_FUNCTION',

'CALL_FUNCTION_EX',

'CALL_FUNCTION_EX_KW',

'CALL_FUNCTION_VAR',

'CALL_FUNCTION_VAR_KW'))

or opname.startswith('CALL_FUNCTION_KW')):

if opname == 'CALL_FUNCTION' and token.attr == 1:

rule = """

if self.version < 3.5:

rule += """

self.addRule(rule, nop_func)

self.custom_classfunc_rule(opname, token, customize, tokens[i+1])

# Note: don't add to custom_ops_processed.

elif opname_base == 'CALL_METHOD':

# PyPy only - DRY with parse2

args_pos, args_kw = self.get_pos_kw(token)

# number of apply equiv arguments:

nak = ( len(opname_base)-len('CALL_METHOD') ) // 3

rule = ('call ::= expr ' +

('pos_arg ' * args_pos) +

('kwarg ' * args_kw) +

'expr ' * nak + opname)

self.add_unique_rule(rule, opname, token.attr, customize)

elif opname == 'CONTINUE':

self.addRule('continue ::= CONTINUE', nop_func)

custom_ops_processed.add(opname)

elif opname == 'CONTINUE_LOOP':

self.addRule('continue ::= CONTINUE_LOOP', nop_func)

custom_ops_processed.add(opname)

elif opname == 'DELETE_ATTR':

self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func)

custom_ops_processed.add(opname)

elif opname == 'DELETE_DEREF':

self.addRule("""

""", nop_func)

custom_ops_processed.add(opname)

elif opname == 'DELETE_SUBSCR':

self.addRule("""

""", nop_func)

custom_ops_processed.add(opname)

elif opname == 'GET_ITER':

self.addRule("""

""", nop_func)

custom_ops_processed.add(opname)

elif opname == 'JUMP_IF_NOT_DEBUG':

v = token.attr

self.addRule("""

""", nop_func)

custom_ops_processed.add(opname)

elif opname == 'LOAD_BUILD_CLASS':

self.custom_build_class_rule(opname, i, token, tokens, customize)

# Note: don't add to custom_ops_processed.

elif opname == 'LOAD_CLASSDEREF':

# Python 3.4+

self.addRule("expr ::= LOAD_CLASSDEREF", nop_func)

custom_ops_processed.add(opname)

elif opname == 'LOAD_CLASSNAME':

self.addRule("expr ::= LOAD_CLASSNAME", nop_func)

custom_ops_processed.add(opname)

elif opname == 'LOAD_DICTCOMP':

if has_get_iter_call_function1:

rule_pat = ("dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "

"GET_ITER CALL_FUNCTION_1")

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

pass

custom_ops_processed.add(opname)

elif opname == 'LOAD_ATTR':

self.addRule("""

""", nop_func)

custom_ops_processed.add(opname)

elif opname == 'LOAD_LISTCOMP':

self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)

custom_ops_processed.add(opname)

elif opname == 'LOAD_SETCOMP':

# Should this be generalized and put under MAKE_FUNCTION?

if has_get_iter_call_function1:

self.addRule("expr ::= set_comp", nop_func)

rule_pat = ("set_comp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr "

"GET_ITER CALL_FUNCTION_1")

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

pass

custom_ops_processed.add(opname)

elif opname == 'LOOKUP_METHOD':

# A PyPy speciality - DRY with parse3

self.addRule("""

""",

nop_func)

custom_ops_processed.add(opname)

elif opname.startswith('MAKE_CLOSURE'):

# DRY with MAKE_FUNCTION

# Note: this probably doesn't handle kwargs proprerly

if opname == 'MAKE_CLOSURE_0' and 'LOAD_DICTCOMP' in self.seen_ops:

# Is there something general going on here?

# Note that 3.6+ doesn't do this, but we'll remove

# this rule in parse36.py

rule = """

self.addRule(rule, nop_func)

args_pos, args_kw, annotate_args = token.attr

# FIXME: Fold test into add_make_function_rule

if self.version < 3.3:

j = 1

else:

j = 2

if is_pypy or (i >= j and tokens[i-j] == 'LOAD_LAMBDA'):

rule_pat = ('mklambda ::= %sload_closure LOAD_LAMBDA %%s%s' %

('pos_arg '* args_pos, opname))

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

if has_get_iter_call_function1:

rule_pat = ("generator_exp ::= %sload_closure load_genexpr %%s%s expr "

"GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

if has_get_iter_call_function1:

if (is_pypy or (i >= j and tokens[i-j] == 'LOAD_LISTCOMP')):

# In the tokens we saw:

# LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or

# LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or

# and have GET_ITER CALL_FUNCTION_1

# Todo: For Pypy we need to modify this slightly

rule_pat = ('listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr '

'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

if (is_pypy or (i >= j and tokens[i-j] == 'LOAD_SETCOMP')):

rule_pat = ('set_comp ::= %sload_closure LOAD_SETCOMP %%s%s expr '

'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

if (is_pypy or (i >= j and tokens[i-j] == 'LOAD_DICTCOMP')):

self.add_unique_rule('dict_comp ::= %sload_closure LOAD_DICTCOMP %s '

'expr GET_ITER CALL_FUNCTION_1' %

('pos_arg '* args_pos, opname),

opname, token.attr, customize)

if args_kw > 0:

kwargs_str = 'kwargs '

else:

kwargs_str = ''

# Note order of kwargs and pos args changed between 3.3-3.4

if self.version <= 3.2:

rule = ('mkfunc ::= %s%sload_closure LOAD_CONST %s'

% (kwargs_str, 'expr ' * args_pos, opname))

elif self.version == 3.3:

rule = ('mkfunc ::= %s%sload_closure LOAD_CONST LOAD_CONST %s'

% (kwargs_str, 'expr ' * args_pos, opname))

elif self.version >= 3.4:

rule = ('mkfunc ::= %s%s load_closure LOAD_CONST LOAD_CONST %s'

% ('expr ' * args_pos, kwargs_str, opname))

self.add_unique_rule(rule, opname, token.attr, customize)

if args_kw == 0:

rule = ('mkfunc ::= %sload_closure load_genexpr %s'

% ('pos_arg ' * args_pos, opname))

self.add_unique_rule(rule, opname, token.attr, customize)

if self.version < 3.4:

rule = ('mkfunc ::= %sload_closure LOAD_CONST %s'

% ('expr ' * args_pos, opname))

self.add_unique_rule(rule, opname, token.attr, customize)

pass

elif opname_base.startswith('MAKE_FUNCTION'):

# DRY with MAKE_CLOSURE

if self.version >= 3.6:

# The semantics of MAKE_FUNCTION in 3.6 are totally different from

# before.

args_pos, args_kw, annotate_args, closure = token.attr

stack_count = args_pos + args_kw + annotate_args

if closure:

if args_pos:

rule = ('mklambda ::= %s%s%s%s' %

('expr ' * stack_count,

'load_closure ' * closure,

'BUILD_TUPLE_1 LOAD_LAMBDA LOAD_CONST ',

opname))

else:

rule = ('mklambda ::= %s%s%s' %

('load_closure ' * closure,

'LOAD_LAMBDA LOAD_CONST ',

opname))

self.add_unique_rule(rule, opname, token.attr, customize)

rule = ('mkfunc ::= %s%s%s%s' %

('expr ' * stack_count,

'load_closure ' * closure,

'LOAD_CONST ' * 2,

opname))

self.add_unique_rule(rule, opname, token.attr, customize)

if has_get_iter_call_function1:

rule_pat = ("generator_exp ::= %sload_genexpr %%s%s expr "

"GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

rule_pat = ("generator_exp ::= %sload_closure load_genexpr %%s%s expr "

"GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

if is_pypy or (i >= 2 and tokens[i-2] == 'LOAD_LISTCOMP'):

if self.version >= 3.6:

# 3.6+ sometimes bundles all of the

# 'exprs' in the rule above into a

# tuple.

rule_pat = ("listcomp ::= load_closure LOAD_LISTCOMP %%s%s "

"expr GET_ITER CALL_FUNCTION_1" % (opname,))

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr "

"GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname))

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

if is_pypy or (i >= 2 and tokens[i-2] == 'LOAD_LAMBDA'):

rule_pat = ('mklambda ::= %s%sLOAD_LAMBDA %%s%s' %

(('pos_arg '* args_pos),

('kwarg '* args_kw),

opname))

self.add_make_function_rule(rule_pat, opname, token.attr, customize)

continue

if self.version < 3.6:

args_pos, args_kw, annotate_args = token.attr

else:

args_pos, args_kw, annotate_args, closure = token.attr

if self.version < 3.3:

j = 1

else:

j = 2

if has_get_iter_call_function1:

rule_pat = ("generator_exp ::= %sload_genexpr %%s%s expr "