#include "Python.h"

#include "Python-ast.h"

#include "grammar.h"

#include "node.h"

#include "pyarena.h"

#include "ast.h"

#include "token.h"

#include "parsetok.h"

#include "graminit.h"

#include <assert.h>

struct compiling {

};

static asdl_seq *seq_for_testlist(struct compiling *, const node *);

static expr_ty ast_for_expr(struct compiling *, const node *);

static stmt_ty ast_for_stmt(struct compiling *, const node *);

static asdl_seq *ast_for_suite(struct compiling *, const node *);

static asdl_seq *ast_for_exprlist(struct compiling *, const node *,

expr_context_ty);

static expr_ty ast_for_testlist(struct compiling *, const node *);

static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);

static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);

static PyObject *parsenumber(struct compiling *, const char *);

static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode);

static PyObject *parsestrplus(struct compiling *, const node *n,

int *bytesmode);

#ifndef LINENO

#define LINENO(n) ((n)->n_lineno)

#define COMP_GENEXP 0

#define COMP_LISTCOMP 1

#define COMP_SETCOMP 2

static identifier

new_identifier(const char* n, PyArena *arena)

{

PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);

Py_UNICODE *u;

if (!id)

return NULL;

u = PyUnicode_AS_UNICODE(id);

/* Check whether there are non-ASCII characters in the

for (; *u; u++) {

if (*u >= 128) {

PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");

PyObject *id2;

if (!m)

return NULL;

id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id);

Py_DECREF(m);

if (!id2)

return NULL;

Py_DECREF(id);

id = id2;

break;

}

}

PyUnicode_InternInPlace(&id);

PyArena_AddPyObject(arena, id);

return id;

}

#define NEW_IDENTIFIER(n) new_identifier(STR(n), c->c_arena)

static int

ast_error(const node *n, const char *errstr)

{

PyObject *u = Py_BuildValue("zi", errstr, LINENO(n));

if (!u)

return 0;

PyErr_SetObject(PyExc_SyntaxError, u);

Py_DECREF(u);

return 0;

}

static void

ast_error_finish(const char *filename)

{

PyObject *type, *value, *tback, *errstr, *loc, *tmp;

long lineno;

assert(PyErr_Occurred());

if (!PyErr_ExceptionMatches(PyExc_SyntaxError))

return;

PyErr_Fetch(&type, &value, &tback);

errstr = PyTuple_GetItem(value, 0);

if (!errstr)

return;

Py_INCREF(errstr);

lineno = PyLong_AsLong(PyTuple_GetItem(value, 1));

if (lineno == -1) {

Py_DECREF(errstr);

return;

}

Py_DECREF(value);

loc = PyErr_ProgramText(filename, lineno);

if (!loc) {

Py_INCREF(Py_None);

loc = Py_None;

}

tmp = Py_BuildValue("(zlOO)", filename, lineno, Py_None, loc);

Py_DECREF(loc);

if (!tmp) {

Py_DECREF(errstr);

return;

}

value = PyTuple_Pack(2, errstr, tmp);

Py_DECREF(errstr);

Py_DECREF(tmp);

if (!value)

return;

PyErr_Restore(type, value, tback);

}

static int

num_stmts(const node *n)

{

int i, l;

node *ch;

switch (TYPE(n)) {

case single_input:

if (TYPE(CHILD(n, 0)) == NEWLINE)

return 0;

else

return num_stmts(CHILD(n, 0));

case file_input:

l = 0;

for (i = 0; i < NCH(n); i++) {

ch = CHILD(n, i);

if (TYPE(ch) == stmt)

l += num_stmts(ch);

}

return l;

case stmt:

return num_stmts(CHILD(n, 0));

case compound_stmt:

return 1;

case simple_stmt:

return NCH(n) / 2; /* Divide by 2 to remove count of semi-colons */

case suite:

if (NCH(n) == 1)

return num_stmts(CHILD(n, 0));

else {

l = 0;

for (i = 2; i < (NCH(n) - 1); i++)

l += num_stmts(CHILD(n, i));

return l;

}

default: {

char buf[128];

sprintf(buf, "Non-statement found: %d %d",

TYPE(n), NCH(n));

Py_FatalError(buf);

}

}

assert(0);

return 0;

}

PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,

PyArena *arena)

{

int i, j, k, num;

asdl_seq *stmts = NULL;

stmt_ty s;

node *ch;

struct compiling c;

if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) {

c.c_encoding = "utf-8";

if (TYPE(n) == encoding_decl) {

#if 0

ast_error(n, "encoding declaration in Unicode string");

goto error;

n = CHILD(n, 0);

}

} else if (TYPE(n) == encoding_decl) {

c.c_encoding = STR(n);

n = CHILD(n, 0);

} else {

/* PEP 3120 */

c.c_encoding = "utf-8";

}

c.c_arena = arena;

c.c_filename = filename;

k = 0;

switch (TYPE(n)) {

case file_input:

stmts = asdl_seq_new(num_stmts(n), arena);

if (!stmts)

return NULL;

for (i = 0; i < NCH(n) - 1; i++) {

ch = CHILD(n, i);

if (TYPE(ch) == NEWLINE)

continue;

REQ(ch, stmt);

num = num_stmts(ch);

if (num == 1) {

s = ast_for_stmt(&c, ch);

if (!s)

goto error;

asdl_seq_SET(stmts, k++, s);

}

else {

ch = CHILD(ch, 0);

REQ(ch, simple_stmt);

for (j = 0; j < num; j++) {

s = ast_for_stmt(&c, CHILD(ch, j * 2));

if (!s)

goto error;

asdl_seq_SET(stmts, k++, s);

}

}

}

return Module(stmts, arena);

case eval_input: {

expr_ty testlist_ast;

/* XXX Why not comp_for here? */

testlist_ast = ast_for_testlist(&c, CHILD(n, 0));

if (!testlist_ast)

goto error;

return Expression(testlist_ast, arena);

}

case single_input:

if (TYPE(CHILD(n, 0)) == NEWLINE) {

stmts = asdl_seq_new(1, arena);

if (!stmts)

goto error;

asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset,

arena));

if (!asdl_seq_GET(stmts, 0))

goto error;

return Interactive(stmts, arena);

}

else {

n = CHILD(n, 0);

num = num_stmts(n);

stmts = asdl_seq_new(num, arena);

if (!stmts)

goto error;

if (num == 1) {

s = ast_for_stmt(&c, n);

if (!s)

goto error;

asdl_seq_SET(stmts, 0, s);

}

else {

/* Only a simple_stmt can contain multiple statements. */

REQ(n, simple_stmt);

for (i = 0; i < NCH(n); i += 2) {

if (TYPE(CHILD(n, i)) == NEWLINE)

break;

s = ast_for_stmt(&c, CHILD(n, i));

if (!s)

goto error;

asdl_seq_SET(stmts, i / 2, s);

}

}

return Interactive(stmts, arena);

}

default:

PyErr_Format(PyExc_SystemError,

"invalid node %d for PyAST_FromNode", TYPE(n));

goto error;

}

error:

ast_error_finish(filename);

return NULL;

}

static operator_ty

get_operator(const node *n)

{

switch (TYPE(n)) {

case VBAR:

return BitOr;

case CIRCUMFLEX:

return BitXor;

case AMPER:

return BitAnd;

case LEFTSHIFT:

return LShift;

case RIGHTSHIFT:

return RShift;

case PLUS:

return Add;

case MINUS:

return Sub;

case STAR:

return Mult;

case SLASH:

return Div;

case DOUBLESLASH:

return FloorDiv;

case PERCENT:

return Mod;

default:

return (operator_ty)0;

}

}

static const char* FORBIDDEN[] = {

"None",

"True",

"False",

"__debug__",

NULL,

};

static int

forbidden_name(expr_ty e, const node *n)

{

const char **p;

assert(PyUnicode_Check(e->v.Name.id));

for (p = FORBIDDEN; *p; p++) {

if (PyUnicode_CompareWithASCIIString(e->v.Name.id, *p) == 0) {

ast_error(n, "assignment to keyword");

return 1;

}

}

return 0;

}

static int

set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)

{

asdl_seq *s = NULL;

/* If a particular expression type can't be used for assign / delete,

const char* expr_name = NULL;

/* The ast defines augmented store and load contexts, but the

assert(ctx != AugStore && ctx != AugLoad);

switch (e->kind) {

case Attribute_kind:

e->v.Attribute.ctx = ctx;

break;

case Subscript_kind:

e->v.Subscript.ctx = ctx;

break;

case Starred_kind:

e->v.Starred.ctx = ctx;

if (!set_context(c, e->v.Starred.value, ctx, n))

return 0;

break;

case Name_kind:

if (ctx == Store) {

if (forbidden_name(e, n))

return 0; /* forbidden_name() calls ast_error() */

}

e->v.Name.ctx = ctx;

break;

case List_kind:

e->v.List.ctx = ctx;

s = e->v.List.elts;

break;

case Tuple_kind:

if (asdl_seq_LEN(e->v.Tuple.elts) == 0)

return ast_error(n, "can't assign to ()");

e->v.Tuple.ctx = ctx;

s = e->v.Tuple.elts;

break;

case Lambda_kind:

expr_name = "lambda";

break;

case Call_kind:

expr_name = "function call";

break;

case BoolOp_kind:

case BinOp_kind:

case UnaryOp_kind:

expr_name = "operator";

break;

case GeneratorExp_kind:

expr_name = "generator expression";

break;

case Yield_kind:

expr_name = "yield expression";

break;

case ListComp_kind:

expr_name = "list comprehension";

break;

case SetComp_kind:

expr_name = "set comprehension";

break;

case DictComp_kind:

expr_name = "dict comprehension";

break;

case Dict_kind:

case Set_kind:

case Num_kind:

case Str_kind:

expr_name = "literal";

break;

case Ellipsis_kind:

expr_name = "Ellipsis";

break;

case Compare_kind:

expr_name = "comparison";

break;

case IfExp_kind:

expr_name = "conditional expression";

break;

default:

PyErr_Format(PyExc_SystemError,

"unexpected expression in assignment %d (line %d)",

e->kind, e->lineno);

return 0;

}

/* Check for error string set by switch */

if (expr_name) {

char buf[300];

PyOS_snprintf(buf, sizeof(buf),

"can't %s %s",

ctx == Store ? "assign to" : "delete",

expr_name);

return ast_error(n, buf);

}

/* If the LHS is a list or tuple, we need to set the assignment

if (s) {

int i;

for (i = 0; i < asdl_seq_LEN(s); i++) {

if (!set_context(c, (expr_ty)asdl_seq_GET(s, i), ctx, n))

return 0;

}

}

return 1;

}

static operator_ty

ast_for_augassign(struct compiling *c, const node *n)

{

REQ(n, augassign);

n = CHILD(n, 0);

switch (STR(n)[0]) {

case '+':

return Add;

case '-':

return Sub;

case '/':

if (STR(n)[1] == '/')

return FloorDiv;

else

return Div;

case '%':

return Mod;

case '<':

return LShift;

case '>':

return RShift;

case '&':

return BitAnd;

case '^':

return BitXor;

case '|':

return BitOr;

case '*':

if (STR(n)[1] == '*')

return Pow;

else

return Mult;

default:

PyErr_Format(PyExc_SystemError, "invalid augassign: %s", STR(n));

return (operator_ty)0;

}

}

static cmpop_ty

ast_for_comp_op(struct compiling *c, const node *n)

{

/* comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'

REQ(n, comp_op);

if (NCH(n) == 1) {

n = CHILD(n, 0);

switch (TYPE(n)) {

case LESS:

return Lt;

case GREATER:

return Gt;

case EQEQUAL: /* == */

return Eq;

case LESSEQUAL:

return LtE;

case GREATEREQUAL:

return GtE;

case NOTEQUAL:

return NotEq;

case NAME:

if (strcmp(STR(n), "in") == 0)

return In;

if (strcmp(STR(n), "is") == 0)

return Is;

default:

PyErr_Format(PyExc_SystemError, "invalid comp_op: %s",

STR(n));

return (cmpop_ty)0;

}

}

else if (NCH(n) == 2) {

/* handle "not in" and "is not" */

switch (TYPE(CHILD(n, 0))) {

case NAME:

if (strcmp(STR(CHILD(n, 1)), "in") == 0)

return NotIn;

if (strcmp(STR(CHILD(n, 0)), "is") == 0)

return IsNot;

default:

PyErr_Format(PyExc_SystemError, "invalid comp_op: %s %s",

STR(CHILD(n, 0)), STR(CHILD(n, 1)));

return (cmpop_ty)0;

}

}

PyErr_Format(PyExc_SystemError, "invalid comp_op: has %d children",

NCH(n));

return (cmpop_ty)0;

}

static asdl_seq *

seq_for_testlist(struct compiling *c, const node *n)

{

/* testlist: test (',' test)* [','] */

asdl_seq *seq;

expr_ty expression;

int i;

assert(TYPE(n) == testlist || TYPE(n) == testlist_comp);

seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);

if (!seq)

return NULL;

for (i = 0; i < NCH(n); i += 2) {

assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == test_nocond);

expression = ast_for_expr(c, CHILD(n, i));

if (!expression)

return NULL;

assert(i / 2 < seq->size);

asdl_seq_SET(seq, i / 2, expression);

}

return seq;

}

static arg_ty

compiler_arg(struct compiling *c, const node *n)

{

identifier name;

expr_ty annotation = NULL;

node *ch;

assert(TYPE(n) == tfpdef || TYPE(n) == vfpdef);

ch = CHILD(n, 0);

name = NEW_IDENTIFIER(ch);

if (!name)

return NULL;

if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == COLON) {

annotation = ast_for_expr(c, CHILD(n, 2));

if (!annotation)

return NULL;

}

return arg(name, annotation, c->c_arena);

#if 0

result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena);

if (!set_context(c, result, Store, n))

return NULL;

return result;

}

static int

handle_keywordonly_args(struct compiling *c, const node *n, int start,

asdl_seq *kwonlyargs, asdl_seq *kwdefaults)

{

PyObject *argname;

node *ch;

expr_ty expression, annotation;

arg_ty arg;

int i = start;

int j = 0; /* index for kwdefaults and kwonlyargs */

if (kwonlyargs == NULL) {

ast_error(CHILD(n, start), "named arguments must follow bare *");

return -1;

}

assert(kwdefaults != NULL);

while (i < NCH(n)) {

ch = CHILD(n, i);

switch (TYPE(ch)) {

case vfpdef:

case tfpdef:

if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {

expression = ast_for_expr(c, CHILD(n, i + 2));

asdl_seq_SET(kwdefaults, j, expression);

i += 2; /* '=' and test */

}

else { /* setting NULL if no default value exists */

asdl_seq_SET(kwdefaults, j, NULL);

}

if (NCH(ch) == 3) {

/* ch is NAME ':' test */

annotation = ast_for_expr(c, CHILD(ch, 2));

if (!annotation) {

ast_error(ch, "expected expression");

goto error;

}

}

else {

annotation = NULL;

}

ch = CHILD(ch, 0);

argname = NEW_IDENTIFIER(ch);

if (!argname)

goto error;

arg = arg(argname, annotation, c->c_arena);

if (!arg)

goto error;

asdl_seq_SET(kwonlyargs, j++, arg);

i += 2; /* the name and the comma */

break;

case DOUBLESTAR:

return i;

default:

ast_error(ch, "unexpected node");

goto error;

}

}

return i;

error:

return -1;

}

static arguments_ty

ast_for_arguments(struct compiling *c, const node *n)

{

/* This function handles both typedargslist (function definition)

int i, j, k, nposargs = 0, nkwonlyargs = 0;

int nposdefaults = 0, found_default = 0;

asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults;

identifier vararg = NULL, kwarg = NULL;

arg_ty arg;

expr_ty varargannotation = NULL, kwargannotation = NULL;

node *ch;

if (TYPE(n) == parameters) {

if (NCH(n) == 2) /* () as argument list */

return arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL,

NULL, c->c_arena);

n = CHILD(n, 1);

}

assert(TYPE(n) == typedargslist || TYPE(n) == varargslist);

/* First count the number of positional args & defaults. The

for (i = 0; i < NCH(n); i++) {

ch = CHILD(n, i);

if (TYPE(ch) == STAR) {

/* skip star */

i++;

if (i < NCH(n) && /* skip argument following star */

(TYPE(CHILD(n, i)) == tfpdef ||

TYPE(CHILD(n, i)) == vfpdef)) {

i++;

}

break;

}

if (TYPE(ch) == DOUBLESTAR) break;

if (TYPE(ch) == vfpdef || TYPE(ch) == tfpdef) nposargs++;

if (TYPE(ch) == EQUAL) nposdefaults++;

}

/* count the number of keyword only args &

for ( ; i < NCH(n); ++i) {

ch = CHILD(n, i);

if (TYPE(ch) == DOUBLESTAR) break;

if (TYPE(ch) == tfpdef || TYPE(ch) == vfpdef) nkwonlyargs++;

}

posargs = (nposargs ? asdl_seq_new(nposargs, c->c_arena) : NULL);

if (!posargs && nposargs)

goto error;

kwonlyargs = (nkwonlyargs ?

asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);

if (!kwonlyargs && nkwonlyargs)

goto error;

posdefaults = (nposdefaults ?

asdl_seq_new(nposdefaults, c->c_arena) : NULL);

if (!posdefaults && nposdefaults)

goto error;

/* The length of kwonlyargs and kwdefaults are same

kwdefaults = (nkwonlyargs ?

asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);

if (!kwdefaults && nkwonlyargs)

goto error;

if (nposargs + nkwonlyargs > 255) {

ast_error(n, "more than 255 arguments");

return NULL;

}

/* tfpdef: NAME [':' test]

i = 0;

j = 0; /* index for defaults */

k = 0; /* index for args */

while (i < NCH(n)) {

ch = CHILD(n, i);

switch (TYPE(ch)) {

case tfpdef:

case vfpdef:

/* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is

/* XXX Should NCH(n) check be made a separate check? */

if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {

expr_ty expression = ast_for_expr(c, CHILD(n, i + 2));

if (!expression)

goto error;

assert(posdefaults != NULL);

asdl_seq_SET(posdefaults, j++, expression);

i += 2;

found_default = 1;

}

else if (found_default) {

ast_error(n,

"non-default argument follows default argument");

goto error;

}

arg = compiler_arg(c, ch);

if (!arg)

goto error;

asdl_seq_SET(posargs, k++, arg);

i += 2; /* the name and the comma */

break;

case STAR:

if (i+1 >= NCH(n)) {

ast_error(CHILD(n, i),

"named arguments must follow bare *");

goto error;

}

ch = CHILD(n, i+1); /* tfpdef or COMMA */

if (TYPE(ch) == COMMA) {

int res = 0;

i += 2; /* now follows keyword only arguments */

res = handle_keywordonly_args(c, n, i,

kwonlyargs, kwdefaults);

if (res == -1) goto error;

i = res; /* res has new position to process */

}

else {

vararg = NEW_IDENTIFIER(CHILD(ch, 0));

if (!vararg)

return NULL;

if (NCH(ch) > 1) {

/* there is an annotation on the vararg */

varargannotation = ast_for_expr(c, CHILD(ch, 2));

}

i += 3;

if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef

|| TYPE(CHILD(n, i)) == vfpdef)) {

int res = 0;

res = handle_keywordonly_args(c, n, i,

kwonlyargs, kwdefaults);

if (res == -1) goto error;

i = res; /* res has new position to process */

}

}

break;

case DOUBLESTAR:

ch = CHILD(n, i+1); /* tfpdef */

assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef);

kwarg = NEW_IDENTIFIER(CHILD(ch, 0));

if (NCH(ch) > 1) {

/* there is an annotation on the kwarg */

kwargannotation = ast_for_expr(c, CHILD(ch, 2));

}

if (!kwarg)

goto error;

i += 3;

break;

default:

PyErr_Format(PyExc_SystemError,

"unexpected node in varargslist: %d @ %d",

TYPE(ch), i);

goto error;

}

}

return arguments(posargs, vararg, varargannotation, kwonlyargs, kwarg,

kwargannotation, posdefaults, kwdefaults, c->c_arena);

error:

Py_XDECREF(vararg);

Py_XDECREF(kwarg);

return NULL;

}

static expr_ty

ast_for_dotted_name(struct compiling *c, const node *n)

{

expr_ty e;

identifier id;

int lineno, col_offset;

int i;

REQ(n, dotted_name);

lineno = LINENO(n);

col_offset = n->n_col_offset;

id = NEW_IDENTIFIER(CHILD(n, 0));

if (!id)

return NULL;

e = Name(id, Load, lineno, col_offset, c->c_arena);

if (!e)

return NULL;

for (i = 2; i < NCH(n); i+=2) {

id = NEW_IDENTIFIER(CHILD(n, i));

if (!id)

return NULL;

e = Attribute(e, id, Load, lineno, col_offset, c->c_arena);

if (!e)

return NULL;

}

return e;

}

static expr_ty

ast_for_decorator(struct compiling *c, const node *n)

{

/* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */

expr_ty d = NULL;

expr_ty name_expr;

REQ(n, decorator);

REQ(CHILD(n, 0), AT);

REQ(RCHILD(n, -1), NEWLINE);

name_expr = ast_for_dotted_name(c, CHILD(n, 1));

if (!name_expr)

return NULL;

if (NCH(n) == 3) { /* No arguments */

d = name_expr;

name_expr = NULL;

}

else if (NCH(n) == 5) { /* Call with no arguments */

d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n),

n->n_col_offset, c->c_arena);

if (!d)

return NULL;

name_expr = NULL;

}

else {

d = ast_for_call(c, CHILD(n, 3), name_expr);

if (!d)

return NULL;

name_expr = NULL;

}

return d;

}

static asdl_seq*

ast_for_decorators(struct compiling *c, const node *n)

{

asdl_seq* decorator_seq;

expr_ty d;

int i;

REQ(n, decorators);

decorator_seq = asdl_seq_new(NCH(n), c->c_arena);

if (!decorator_seq)

return NULL;

for (i = 0; i < NCH(n); i++) {

d = ast_for_decorator(c, CHILD(n, i));

if (!d)

return NULL;

asdl_seq_SET(decorator_seq, i, d);

}

return decorator_seq;

}

static stmt_ty

ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)

{

/* funcdef: 'def' NAME parameters ['->' test] ':' suite */

identifier name;

arguments_ty args;

asdl_seq *body;

expr_ty returns = NULL;

int name_i = 1;

REQ(n, funcdef);

name = NEW_IDENTIFIER(CHILD(n, name_i));

if (!name)

return NULL;