luapy (15) lua parser

212180 ワード

1.parse exp
from lua_token import TokenKind
import lua_exp
from optimizer import Optimizer
from lua_value import LuaValue


class ExpParser:
    @staticmethod
    def parse_exp_list(lexer):
        exps = [ExpParser.parse_exp(lexer)]
        while lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            exps.append(ExpParser.parse_exp(lexer))
        return exps

    @staticmethod
    def parse_exp(lexer):
        return ExpParser.parse_exp12(lexer)

    # x or y
    @staticmethod
    def parse_exp12(lexer):
        exp = ExpParser.parse_exp11(lexer)
        while lexer.look_ahead() == TokenKind.OP_OR:
            line, op, _ = lexer.get_next_token()
            lor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp11(lexer))
            exp = Optimizer.optimize_logical_or(lor)
        return exp

    # x and y
    @staticmethod
    def parse_exp11(lexer):
        exp = ExpParser.parse_exp10(lexer)
        while lexer.look_ahead() == TokenKind.OP_AND:
            line, op, _ = lexer.get_next_token()
            lor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp10(lexer))
            exp = Optimizer.optimize_logical_and(lor)
        return exp

    # compare
    @staticmethod
    def parse_exp10(lexer):
        exp = ExpParser.parse_exp9(lexer)
        while True:
            kind = lexer.look_ahead()
            if kind in (TokenKind.OP_LT, TokenKind.OP_GT, TokenKind.OP_NE,
                        TokenKind.OP_LE, TokenKind.OP_GE, TokenKind.OP_EQ):
                line, op, _ = lexer.get_next_token()
                exp = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp9(lexer))
            else:
                return exp

    # x | y
    @staticmethod
    def parse_exp9(lexer):
        exp = ExpParser.parse_exp8(lexer)
        while lexer.look_ahead() == TokenKind.OP_BOR:
            line, op, _ = lexer.get_next_token()
            bor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp8(lexer))
            exp = Optimizer.optimize_bitwise_binary_op(bor)
        return exp

    # x ~ y
    @staticmethod
    def parse_exp8(lexer):
        exp = ExpParser.parse_exp7(lexer)
        while lexer.look_ahead() == TokenKind.OP_BXOR:
            line, op, _ = lexer.get_next_token()
            bor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp8(lexer))
            exp = Optimizer.optimize_bitwise_binary_op(bor)
        return exp

    # x & y
    @staticmethod
    def parse_exp7(lexer):
        exp = ExpParser.parse_exp6(lexer)
        while lexer.look_ahead() == TokenKind.OP_BAND:
            line, op, _ = lexer.get_next_token()
            bor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp8(lexer))
            exp = Optimizer.optimize_bitwise_binary_op(bor)
        return exp

    # shift
    @staticmethod
    def parse_exp6(lexer):
        exp = ExpParser.parse_exp5(lexer)
        if lexer.look_ahead() in (TokenKind.OP_SHL, TokenKind.OP_SHR):
            line, op, _ = lexer.get_next_token()
            shx = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp5(lexer))
            exp = Optimizer.optimize_bitwise_binary_op(shx)
        else:
            return exp
        return exp

    # a .. b
    @staticmethod
    def parse_exp5(lexer):
        exp = ExpParser.parse_exp4(lexer)
        if lexer.look_ahead() != TokenKind.OP_CONCAT:
            return exp

        line = 0
        exps = []
        while lexer.look_ahead() == TokenKind.OP_CONCAT:
            line, _, _ = lexer.get_next_token()
            exps.append(ExpParser.parse_exp4(lexer))
        return lua_exp.ConcatExp(line, exps)

    # x +/- y
    @staticmethod
    def parse_exp4(lexer):
        exp = ExpParser.parse_exp3(lexer)
        while True:
            if lexer.look_ahead() in (TokenKind.OP_ADD, TokenKind.OP_SUB):
                line, op, _ = lexer.get_next_token()
                arith = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp3(lexer))
                exp = Optimizer.optimize_arith_binary_op(arith)
            else:
                break
        return exp

    # *, %, /, //
    @staticmethod
    def parse_exp3(lexer):
        exp = ExpParser.parse_exp2(lexer)
        while True:
            if lexer.look_ahead() in (TokenKind.OP_MUL, TokenKind.OP_MOD, TokenKind.OP_DIV, TokenKind.OP_IDIV):
                line, op, _ = lexer.get_next_token()
                arith = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp2(lexer))
                exp = Optimizer.optimize_arith_binary_op(arith)
            else:
                break
        return exp

    # unary
    @staticmethod
    def parse_exp2(lexer):
        if lexer.look_ahead() in (TokenKind.OP_UNM, TokenKind.OP_BNOT, TokenKind.OP_LEN, TokenKind.OP_NOT):
            line, op, _ = lexer.get_next_token()
            exp = lua_exp.UnopExp(line, op, ExpParser.parse_exp2(lexer))
            return Optimizer.optimize_unary_op(exp)
        return ExpParser.parse_exp1(lexer)

    # x ^ y
    @staticmethod
    def parse_exp1(lexer):
        exp = ExpParser.parse_exp0(lexer)
        if lexer.look_ahead() == TokenKind.OP_POW:
            line, op, _ = lexer.get_next_token()
            exp = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp2(lexer))
        return Optimizer.optimize_pow(exp)

    @staticmethod
    def parse_exp0(lexer):
        kind = lexer.look_ahead()
        if kind == TokenKind.VARARG:
            line, _, _ = lexer.get_next_token()
            return lua_exp.VarArgExp(line)
        if kind == TokenKind.KW_NIL:
            line, _, _ = lexer.get_next_token()
            return lua_exp.NilExp(line)
        if kind == TokenKind.KW_TRUE:
            line, _, _ = lexer.get_next_token()
            return lua_exp.TrueExp(line)
        if kind == TokenKind.KW_FALSE:
            line, _, _ = lexer.get_next_token()
            return lua_exp.FalseExp(line)
        if kind == TokenKind.STRING:
            line, _, token = lexer.get_next_token()
            return lua_exp.StringExp(line, token)
        if kind == TokenKind.NUMBER:
            return ExpParser.parse_number_exp(lexer)
        if kind == TokenKind.SEP_LCURLY:
            return ExpParser.parse_table_constructor_exp(lexer)
        if kind == TokenKind.KW_FUNCTION:
            lexer.get_next_token()
            return ExpParser.parse_func_def_exp(lexer)
        return ExpParser.parse_prefix_exp(lexer)

    @staticmethod
    def parse_number_exp(lexer):
        line, _, token = lexer.get_next_token()
        i = LuaValue.parse_integer(token)
        if i is not None:
            return lua_exp.IntegerExp(line, i)
        f = LuaValue.parse_float(token)
        if f is not None:
            return lua_exp.FloatExp(line, f)
        raise Exception('not a number: ' + token)

    # functiondef::= function funcbody
    # funcbody::= '(' [parlist] ')' block end
    @staticmethod
    def parse_func_def_exp(lexer):
        from parser import Parser
        line = lexer.get_line()
        lexer.get_next_token_of_kind(TokenKind.SEP_LPAREN)
        par_list, is_var_arg = ExpParser.parse_par_list(lexer)
        lexer.get_next_token_of_kind(TokenKind.SEP_RPAREN)
        block = Parser.parse_block(lexer)
        last_line, _ = lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_exp.FuncDefExp(line, last_line, par_list, is_var_arg, block)

    # [parlist]
    # parlist ::= namelist [',' '...'] | '...'
    @staticmethod
    def parse_par_list(lexer):
        kind = lexer.look_ahead()
        if kind == TokenKind.SEP_RPAREN:
            return None, False
        if kind == TokenKind.VARARG:
            return None, True

        _, name = lexer.get_next_identifier()
        names = [name]
        is_var_arg = False
        while lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            if lexer.look_ahead() == TokenKind.IDENTIFIER:
                _, name = lexer.get_next_identifier()
                names.append(name)
            else:
                lexer.get_next_token_of_kind(TokenKind.VARARG)
                is_var_arg = True
                break
        return names, is_var_arg

    # tableconstructor ::= '{' [fieldlist] '}'
    @staticmethod
    def parse_table_constructor_exp(lexer):
        line = lexer.get_line()
        lexer.get_next_token_of_kind(TokenKind.SEP_LCURLY)
        key_exps, val_exps = ExpParser.parse_field_list(lexer)
        lexer.get_next_token_of_kind(TokenKind.SEP_RCURLY)
        last_line = lexer.get_line()
        return lua_exp.TableConstructorExp(line, last_line, key_exps, val_exps)

    # fieldlist ::= field {fieldsep field} [fieldsep]
    @staticmethod
    def parse_field_list(lexer):
        ks = []
        vs = []
        if lexer.look_ahead != TokenKind.SEP_RCURLY:
            k, v = ExpParser.parse_field(lexer)
            ks.append(k)
            vs.append(v)

            while ExpParser.is_field_sep(lexer.look_ahead()):
                lexer.get_next_token()
                if lexer.look_ahead() != TokenKind.SEP_RCURLY:
                    k, v = ExpParser.parse_field(lexer)
                    ks.append(k)
                    vs.append(v)
                else:
                    break
        return ks, vs

    # fieldsep ::= ',' | ';'
    @staticmethod
    def is_field_sep(kind):
        return kind in (TokenKind.SEP_COMMA, TokenKind.SEP_SEMI)

    # field ::= '[' exp ']' '=' exp | Name '=' exp | exp
    @staticmethod
    def parse_field(lexer):
        if lexer.look_ahead() == TokenKind.SEP_LBRACK:
            lexer.get_next_token()
            k = ExpParser.parse_exp(lexer)
            lexer.get_next_token_of_kind(TokenKind.SEP_RBRACK)
            lexer.get_next_token_of_kind(TokenKind.OP_ASSIGN)
            v = ExpParser.parse_exp(lexer)
            return k, v

        exp = ExpParser.parse_exp(lexer)
        if isinstance(exp, lua_exp.NameExp):
            if lexer.look_ahead() == TokenKind.OP_ASSIGN:
                lexer.get_next_token()
                k = lua_exp.StringExp(exp.line, exp.name)
                v = ExpParser.parse_exp(lexer)
                return k, v

        return None, exp

    # prefixexp::= var | functioncall | '(' exp ')"
    # var::= Name | prefixexp '[' exp ']' | prefixexp '.' Name
    # functioncall ::=  prefixexp
    # args | prefixexp ':' Name
    # args

    # prefixexp::= Name
    #   | '(' exp ')'
    #   | prefixexp '[' exp ']'
    #   | prefixexp '.' Name
    #   | prefixexp[':' Name] args
    @staticmethod
    def parse_prefix_exp(lexer):
        if lexer.look_ahead() == TokenKind.IDENTIFIER:
            line, name = lexer.get_next_identifier()
            exp = lua_exp.NameExp(line, name)
        else:
            exp = ExpParser.parse_parens_exp(lexer)
        return ExpParser.finish_prefix_exp(lexer, exp)

    @staticmethod
    def parse_parens_exp(lexer):
        lexer.get_next_token_of_kind(TokenKind.SEP_LPAREN)
        exp = ExpParser.parse_exp(lexer)
        lexer.get_next_token_of_kind(TokenKind.SEP_RPAREN)

        if isinstance(exp, lua_exp.VarArgExp) or isinstance(exp, lua_exp.FuncCallExp) or \
                isinstance(exp, lua_exp.NameExp) or isinstance(exp, lua_exp.TableAccessExp):
            return lua_exp.ParensExp(exp)
        return exp

    @staticmethod
    def finish_prefix_exp(lexer, exp):
        while True:
            kind = lexer.look_ahead()
            if kind == TokenKind.SEP_LBRACK:
                lexer.get_next_token()
                key_exp = ExpParser.parse_exp(lexer)
                lexer.get_next_token_of_kind(TokenKind.SEP_RBRACK)
                exp = lua_exp.TableAccessExp(lexer.get_line(), exp, key_exp)
            elif kind == TokenKind.SEP_DOT:
                lexer.get_next_token()
                line, name = lexer.get_next_identifier()
                key_exp = lua_exp.StringExp(line, name)
                exp = lua_exp.TableAccessExp(line, exp, key_exp)
            elif kind in (TokenKind.SEP_COLON, TokenKind.SEP_LPAREN, TokenKind.SEP_LCURLY, TokenKind.STRING):
                exp = ExpParser.finish_func_call_exp(lexer, exp)
            return exp

    # functioncall ::= prefixexp args | prefixexp ':' Name args
    @staticmethod
    def finish_func_call_exp(lexer, prefix_exp):
        name_exp = ExpParser.parse_name_exp(lexer)
        line = lexer.get_line()
        args = ExpParser.parse_args(lexer)
        last_line = lexer.get_line()
        return lua_exp.FuncCallExp(line, last_line, prefix_exp, name_exp, args)

    @staticmethod
    def parse_name_exp(lexer):
        if lexer.look_ahead() == TokenKind.SEP_COLON:
            lexer.get_next_token()
            line, name = lexer.get_next_identifier()
            return lua_exp.StringExp(line, name)
        return None

    # args ::= '(' [explist] ')' | tableconstructor | LitreralString
    @staticmethod
    def parse_args(lexer):
        args = []
        kind = lexer.look_ahead()
        if kind == TokenKind.SEP_LPAREN:    # '(' [explist] ')'
            lexer.get_next_token()
            if lexer.look_ahead() != TokenKind.SEP_RPAREN:
                args = ExpParser.parse_exp_list(lexer)
            lexer.get_next_token_of_kind(TokenKind.SEP_RPAREN)
        elif kind == TokenKind.SEP_LCURLY:  # '{' [fieldlist] '}'
            args = [ExpParser.parse_table_constructor_exp(lexer)]
        else:                               # LiteralString
            line, s = lexer.get_next_token_of_kind(TokenKind.STRING)
            args = [lua_exp.StringExp(line, s)]
        return args


2.parse stat
from lua_token import TokenKind
import lua_stat
import lua_exp
from exp_parser import ExpParser
from parser import Parser


"""
stat ::=  ‘;’
    | break
    | ‘::’ Name ‘::’
    | goto Name
    | do block end
    | while exp do block end
    | repeat block until exp
    | if exp then block {elseif exp then block} [else block] end
    | for Name ‘=’ exp ‘,’ exp [‘,’ exp] do block end
    | for namelist in explist do block end
    | function funcname funcbody
    | local function Name funcbody
    | local namelist [‘=’ explist]
    | varlist ‘=’ explist
    | functioncall
"""


class StatParser:
    @staticmethod
    def parse_stat(lexer):
        kind = lexer.look_ahead()
        if kind == TokenKind.SEP_SEMI:
            return StatParser.parse_empty_stat(lexer)
        if kind == TokenKind.KW_BREAK:
            return StatParser.parse_break_stat(lexer)
        if kind == TokenKind.SEP_LABEL:
            return StatParser.parse_label_stat(lexer)
        if kind == TokenKind.KW_GOTO:
            return StatParser.parse_goto_stat(lexer)
        if kind == TokenKind.KW_DO:
            return StatParser.parse_do_stat(lexer)
        if kind == TokenKind.KW_WHILE:
            return StatParser.parse_while_stat(lexer)
        if kind == TokenKind.KW_REPEAT:
            return StatParser.parse_repeat_stat(lexer)
        if kind == TokenKind.KW_IF:
            return StatParser.parse_if_stat(lexer)
        if kind == TokenKind.KW_FOR:
            return StatParser.parse_for_stat(lexer)
        if kind == TokenKind.KW_FUNCTION:
            return StatParser.parse_func_def_stat(lexer)
        if kind == TokenKind.KW_LOCAL:
            return StatParser.parse_local_assign_or_func_def_stat(lexer)
        return StatParser.parse_assign_or_func_call_stat(lexer)

    # ;
    @staticmethod
    def parse_empty_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.SEP_SEMI)
        return lua_stat.EmptyStat(lexer.get_line())

    # break
    @staticmethod
    def parse_break_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_BREAK)
        return lua_stat.BreakStat(lexer.get_line())

    # '::' Name '::'
    @staticmethod
    def parse_label_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.SEP_LABEL)
        _, name = lexer.get_next_identifier()
        lexer.get_next_token_of_kind(TokenKind.SEP_LABEL)
        return lua_stat.LabelStat(name)

    # goto Name
    @staticmethod
    def parse_goto_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_GOTO)
        _, name = lexer.get_next_identifier()
        return lua_stat.GotoStat(name)

    # do block end
    @staticmethod
    def parse_do_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_DO)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_stat.DoStat(block)

    # while exp do block end
    @staticmethod
    def parse_while_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_WHILE)
        exp = ExpParser.parse_exp(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_DO)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_stat.WhileStat(exp, block)

    # repeat block until exp
    @staticmethod
    def parse_repeat_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_REPEAT)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_UNTIL)
        exp = ExpParser.parse_exp(lexer)
        return lua_stat.RepeatStat(block, exp)

    # if exp then block {elseif exp then block} [else block] end
    @staticmethod
    def parse_if_stat(lexer):
        exps = []
        blocks = []

        lexer.get_next_token_of_kind(TokenKind.KW_IF)
        exps.append(ExpParser.parse_exp(lexer))
        lexer.get_next_token_of_kind(TokenKind.KW_THEN)
        blocks.append(Parser.parse_block(lexer))

        while lexer.look_ahead() == TokenKind.KW_ELSEIF:
            lexer.get_next_token()
            exps.append(ExpParser.parse_exp(lexer))
            lexer.get_next_token_of_kind(TokenKind.KW_THEN)
            blocks.append(Parser.parse_block(lexer))

        if lexer.look_ahead() == TokenKind.KW_ELSE:
            lexer.get_next_token()
            exps.append(lua_exp.TrueExp(lexer.get_line()))
            blocks.append(Parser.parse_block(lexer))

        lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_stat.IfStat(exps, blocks)

    # for Name '=' exp ',' exp [',' exp] do block end
    # for namelist in explist do block end
    @staticmethod
    def parse_for_stat(lexer):
        line_of_for, _ = lexer.get_next_token_of_kind(TokenKind.KW_FOR)
        _, name = lexer.get_next_identifier()
        if lexer.look_ahead() == TokenKind.OP_ASSIGN:
            return StatParser.finish_for_num_stat(lexer, line_of_for, name)
        else:
            return StatParser.finish_for_in_stat(lexer, name)

    # for Name '=' exp ',' exp [',' exp] do block end
    @staticmethod
    def finish_for_num_stat(lexer, line_of_for, var_name):
        lexer.get_next_token_of_kind(TokenKind.OP_ASSIGN)
        init_exp = ExpParser.parse_exp(lexer)
        lexer.get_next_token_of_kind(TokenKind.SEP_COMMA)
        limit_exp = ExpParser.parse_exp(lexer)

        if lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            step_exp = ExpParser.parse_exp(lexer)
        else:
            step_exp = lua_exp.IntegerExp(lexer.get_line(), 1)

        line_of_do, _ = lexer.get_next_token_of_kind(TokenKind.KW_DO)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_END)

        return lua_stat.ForNumStat(line_of_for, line_of_do, var_name, init_exp, limit_exp, step_exp, block)

    # for Name '=' exp ',' exp [',' exp] do block end
    @staticmethod
    def finish_for_in_stat(lexer, name):
        name_list = StatParser.finish_name_list(lexer, name)
        lexer.get_next_token_of_kind(TokenKind.KW_IN)
        exp_list = ExpParser.parse_exp_list(lexer)
        line_of_do, _ = lexer.get_next_token_of_kind(TokenKind.KW_DO)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_stat.ForInStat(line_of_do, name_list, exp_list, block)

    # namelist ::= Name {',' Name}
    @staticmethod
    def finish_name_list(lexer, name0):
        names = [name0]
        while lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            _, name = lexer.get_next_identifier()
            names.append(name)
        return names

    #
    @staticmethod
    def parse_func_def_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_FUNCTION)
        fn_exp, has_colon = StatParser.parse_func_name(lexer)
        fd_exp = ExpParser.parse_func_def_exp(lexer)
        if has_colon:
            fd_exp.par_list.insert(0, 'self')

        return lua_stat.AssignStat(fd_exp.line, fn_exp, fd_exp)

    # local function Name funcbody
    # local namelist ['=' explist]
    @staticmethod
    def parse_local_assign_or_func_def_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_LOCAL)
        if lexer.look_ahead() == TokenKind.KW_FUNCTION:
            return StatParser.finish_local_func_def_stat(lexer)
        else:
            return StatParser.finish_local_var_decl_stat(lexer)

    """
    http: // www.lua.org / manual / 5.3 / manual.html  # 3.4.11

    function f() end = > f = function() end
    function t.a.b.c.f() end = > t.a.b.c.f = function() end
    function t.a.b.c: f() end = > t.a.b.c.f = function(self) end
    local function f() end = > local f; f = function() end

    The statement 'local function f() body end' 
    translates to 'local f; f = function() body end`
    not to `local f = function() body end`
    (This only makes a difference when the body of the function
    contains references to f.)
    """

    # local function Name funcbody
    @staticmethod
    def finish_local_func_def_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_FUNCTION)
        _, name = lexer.get_next_identifier()
        fd_exp = ExpParser.parse_func_def_exp(lexer)
        return lua_stat.LocalFuncDefStat(name, fd_exp)

    # local namelist ['=' explist]
    @staticmethod
    def finish_local_var_decl_stat(lexer):
        _, name0 = lexer.get_next_identifier()
        name_list = StatParser.finish_name_list(lexer, name0)
        exp_list = []
        if lexer.look_ahead() == TokenKind.OP_ASSIGN:
            lexer.get_next_token()
            exp_list = ExpParser.parse_exp_list(lexer)
        last_line = lexer.get_line()
        return lua_stat.LocalVarDeclStat(last_line, name_list, exp_list)

    # varlist '=' explist
    # functioncall
    @staticmethod
    def parse_assign_or_func_call_stat(lexer):
        prefix_exp = ExpParser.parse_prefix_exp(lexer)
        if isinstance(prefix_exp, lua_exp.FuncCallExp):
            return prefix_exp
        else:
            return StatParser.parse_assign_stat(lexer, prefix_exp)

    # varlist '=' explist
    @staticmethod
    def parse_assign_stat(lexer, var0):
        var_list = StatParser.finish_var_list(lexer, var0)
        lexer.get_next_token_of_kind(TokenKind.OP_ASSIGN)
        exp_list = ExpParser.parse_exp_list(lexer)
        last_line = lexer.get_line()
        return lua_stat.AssignStat(last_line, var_list, exp_list)

    # varlist ::= var {',' var}
    @staticmethod
    def finish_var_list(lexer, var0):
        var_list = [StatParser.check_var(lexer, var0)]
        while lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            exp = ExpParser.parse_prefix_exp(lexer)
            var_list.append(StatParser.check_var(lexer, exp))
        return var_list

    # var ::= Name | prefixexp '[' exp ']' | prefixexp '.' Name
    @staticmethod
    def check_var(lexer, exp):
        if isinstance(exp, lua_exp.NameExp) or isinstance(exp, lua_exp.TableAccessExp):
            return exp
        lexer.get_next_token_of_kind(-1)
        raise Exception('unreachable!')

    # function funcname funcbody
    # funcname ::= Name {'.' Name} [':' Name]
    # funcbody ::= '(' [parlist] ')' block end
    # parlist ::= namelist [',' '...'] | '...'
    # namelist ::= Name {',' Name}
    @staticmethod
    def parse_func_def_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_FUNCTION)
        fn_exp, has_colon = StatParser.parse_func_name(lexer)
        fd_exp = ExpParser.parse_func_def_exp(lexer)
        if has_colon:
            fd_exp.insert(0, 'self')
        return lua_stat.AssignStat(fd_exp.line, [fd_exp], [fn_exp])

    # funcname ::= Name {'.' Name} [':' Name]
    @staticmethod
    def parse_func_name(lexer):
        line, name = lexer.get_next_identifier()
        exp = lua_exp.NameExp(line, name)
        has_colon = False

        while lexer.look_ahead() == TokenKind.SEP_DOT:
            lexer.get_next_token()
            line, name = lexer.get_next_identifier()
            idx = lua_exp.StringExp(line, name)
            exp = lua_exp.TableAccessExp(line, exp, idx)

        if lexer.look_ahead() == TokenKind.SEP_COLON:
            lexer.get_next_token()
            line, name = lexer.get_next_identifier()
            idx = lua_exp.StringExp(line, name)
            exp = lua_exp.TableAccessExp(line, exp, idx)
            has_colon = True

        return exp, has_colon


3.optimizer
from lua_token import TokenKind
import lua_exp
from lua_value import LuaValue


class Optimizer:
    # or
    @staticmethod
    def optimize_logical_or(exp):
        if Optimizer.is_true(exp.exp1):
            # true or x => true
            return exp.exp1
        if Optimizer.is_false(exp.exp1) and not Optimizer.is_var_arg_or_func_call(exp.exp2):
            # false or x => x
            return exp.exp2
        return exp

    # and
    @staticmethod
    def optimize_logical_and(exp):
        if Optimizer.is_false(exp.exp1):
            # false and x => false
            return exp.exp1
        if Optimizer.is_true(exp.exp1) and not Optimizer.is_var_arg_or_func_call(exp.exp2):
            # true and x => x
            return exp.exp2
        return exp

    # & | ~ << >>
    @staticmethod
    def optimize_bitwise_binary_op(exp):
        i, oki = Optimizer.cast_to_int(exp.exp1)
        j, okj = Optimizer.cast_to_int(exp.exp2)
        if oki and okj:
            if exp.op == TokenKind.OP_BAND:
                return lua_exp.IntegerExp(exp.line, i & j)
            if exp.op == TokenKind.OP_BOR:
                return lua_exp.IntegerExp(exp.line, i | j)
            if exp.op == TokenKind.OP_BXOR:
                return lua_exp.IntegerExp(exp.line, i ^ j)
            if exp.op == TokenKind.OP_SHL:
                return lua_exp.IntegerExp(exp.line, i << j)
            if exp.op == TokenKind.OP_SHR:
                return lua_exp.IntegerExp(exp.line, i >> j)
        return exp

    # + - * / // %
    @staticmethod
    def optimize_arith_binary_op(exp):
        if isinstance(exp.exp1, lua_exp.IntegerExp):
            if isinstance(exp.exp2, lua_exp.IntegerExp):
                if exp.op == TokenKind.OP_ADD:
                    return lua_exp.IntegerExp(exp.line, exp.exp1.val + exp.exp2.val)
                if exp.op == TokenKind.OP_SUB:
                    return lua_exp.IntegerExp(exp.line, exp.exp1.val - exp.exp2.val)
                if exp.op == TokenKind.OP_MUL:
                    return lua_exp.IntegerExp(exp.line, exp.exp1.val * exp.exp2.val)
                if exp.op == TokenKind.OP_IDIV:
                    if exp.exp2.val != 0:
                        return lua_exp.IntegerExp(exp.line, exp.exp1.val // exp.exp2.val)
                if exp.op == TokenKind.OP_MOD:
                    if exp.exp2.val != 0:
                        return lua_exp.IntegerExp(exp.line, exp.exp1.val % exp.exp2.val)

        f, okf = Optimizer.cast_to_float(exp.exp1)
        g, okg = Optimizer.cast_to_float(exp.exp2)
        if okf and okg:
            if exp.op == TokenKind.OP_ADD:
                return lua_exp.IntegerExp(exp.line, f + g)
            if exp.op == TokenKind.OP_SUB:
                return lua_exp.IntegerExp(exp.line, f - g)
            if exp.op == TokenKind.OP_MUL:
                return lua_exp.IntegerExp(exp.line, f * g)
            if exp.op == TokenKind.OP_DIV:
                if g != 0:
                    return lua_exp.IntegerExp(exp.line, f / g)
            if exp.op == TokenKind.OP_IDIV:
                if g != 0:
                    return lua_exp.IntegerExp(exp.line, f // g)
            if exp.op == TokenKind.OP_MOD:
                if g != 0:
                    return lua_exp.IntegerExp(exp.line, f % g)
            if exp.op == TokenKind.OP_POW:
                return lua_exp.IntegerExp(exp.line, f ** g)
        return exp

    # ^
    @staticmethod
    def optimize_pow(exp):
        if isinstance(exp, lua_exp.BinopExp):
            if exp.op == TokenKind.OP_POW:
                exp.exp2 = Optimizer.optimize_pow(exp.exp2)
            return Optimizer.optimize_arith_binary_op(exp)
        return exp

    # - not ~
    @staticmethod
    def optimize_unary_op(exp):
        if exp.op == TokenKind.OP_UNM:
            return Optimizer.optimize_unm(exp)
        if exp.op == TokenKind.OP_NOT:
            return Optimizer.optimize_not(exp)
        if exp.op == TokenKind.OP_BNOT:
            return Optimizer.optimize_bnot(exp)
        return exp

    @staticmethod
    def optimize_unm(exp):
        if isinstance(exp.exp, lua_exp.IntegerExp):
            exp.exp.val = -exp.exp.val
            return exp.exp
        if isinstance(exp.exp, lua_exp.FloatExp):
            if exp.exp.val != 0:
                exp.exp.val = -exp.exp.val
                return exp.val
        return exp

    # not
    @staticmethod
    def optimize_not(exp):
        if isinstance(exp.exp, lua_exp.NilExp) or isinstance(exp.exp, lua_exp.FalseExp):
            return lua_exp.TrueExp(exp.line)
        if isinstance(exp.exp, lua_exp.TrueExp) or isinstance(exp.exp, lua_exp.FloatExp) or \
           isinstance(exp.exp, lua_exp.StringExp):
            return lua_exp.FalseExp(exp.line)
        return exp

    # ~
    @staticmethod
    def optimize_bnot(exp):
        if isinstance(exp.exp, lua_exp.IntegerExp):
            exp.exp.val = ~exp.exp.val
            return exp.exp.val
        if isinstance(exp.exp, lua_exp.FloatExp):
            i = LuaValue.float2integer(exp.exp.val)
            if i is not None:
                return lua_exp.IntegerExp(exp.exp.line, ~i)
        return exp

    # false
    @staticmethod
    def is_false(exp):
        if isinstance(exp, lua_exp.FalseExp) or isinstance(exp, lua_exp.NilExp):
            return True
        return False

    # true
    @staticmethod
    def is_true(exp):
        if isinstance(exp, lua_exp.TrueExp) or isinstance(exp, lua_exp.IntegerExp) or \
           isinstance(exp, lua_exp.FloatExp) or isinstance(exp, lua_exp.StringExp):
            return True
        return False

    @staticmethod
    def is_var_arg_or_func_call(exp):
        if isinstance(exp, lua_exp.VarArgExp) or isinstance(exp, lua_exp.FuncCallExp):
            return True
        return False

    @staticmethod
    def cast_to_int(exp):
        if isinstance(exp, lua_exp.IntegerExp):
            return exp.val, True
        if isinstance(exp, lua_exp.FloatExp):
            i = LuaValue.float2integer(exp.val)
            return i, i is not None
        return 0, False

    @staticmethod
    def cast_to_float(exp):
        if isinstance(exp, lua_exp.IntegerExp):
            return float(exp.val), True
        if isinstance(exp, lua_exp.FloatExp):
            return exp.val, True
        return 0, False


4.parser
from lua_token import TokenKind
from exp_parser import ExpParser


class Block:
    # block ::= {stat} [retstat]
    def __init__(self, lexer):
        self.stats = Block.parse_stats(lexer)
        self.ret_exps = Block.parse_ret_exps(lexer)
        self.last_line = lexer.get_line()

    def __str__(self):
        s = '"LastLine": ' + str(self.last_line) + ',
'
s += '"Stats": ' + '[' for stat in self.stats: s += '{
'
for line in str(stat).split('
'
): if len(line): s += ' ' + line + '
'
s += '}' s += ']
'
s += '"RetExps": ' + '
'
if self.ret_exps: for exp in self.ret_exps: for l in str(exp).split('
'
): if len(l): s += ' ' + l + '
'
return s @staticmethod def parse_stats(lexer): from stat_parser import StatParser stats = [] while not TokenKind.is_return_or_block_end(lexer.look_ahead()): stat = StatParser.parse_stat(lexer) if stat is not None: stats.append(stat) return stats # retstat ::= return [explist] [';'] # explist ::= exp {',' exp} @staticmethod def parse_ret_exps(lexer): if lexer.look_ahead() != TokenKind.KW_RETURN: return None lexer.get_next_token() kind = lexer.look_ahead() if kind in (TokenKind.EOF, TokenKind.KW_END, TokenKind.KW_ELSE, TokenKind.KW_ELSEIF, TokenKind.KW_UNTIL): return [] if kind == TokenKind.SEP_SEMI: lexer.get_next_token() return [] exps = ExpParser.parse_exp_list(lexer) if lexer.look_ahead() == TokenKind.SEP_SEMI: lexer.get_next_token() return exps class Parser: def __init__(self): pass @staticmethod def parse_block(lexer): return Block(lexer)

5.test parser
import sys
from parser import Parser
from lexer import Lexer


def test_parser(chunk, chunkname):
    parser = Parser()
    lexer = Lexer(chunk, chunkname)
    ast = parser.parse_block(lexer)
    print(ast)


def main(file_name):
    with open(file_name, 'r') as f:
        data = f.read()
        test_parser(data, file_name)


if __name__ == '__main__':
    if len(sys.argv) == 2:
        main(sys.argv[1])
    else:
        print('Error argument')

  • test hello world
  • print("Hello")
    print("World")
    
    "LastLine": 2,
    "Stats": [{
      "Line": 1,
      "LastLine": 1,
      "PrefixExp": {
        "Line": 1
        "Name": "print"
      },
      "NameExp": None,
      "Args": [{
        "Line": 1
        "Str": "Hello"
      }]
    }{
      "Line": 2,
      "LastLine": 2,
      "PrefixExp": {
        "Line": 2
        "Name": "print"
      },
      "NameExp": None,
      "Args": [{
        "Line": 2
        "Str": "World"
      }]
    }]
    "RetExps": 
      nil
    

    7.test assign
    a = 1
    b = 2
    c = a+b
    d = a & b
    
    "LastLine": 7,
    "Stats": [{
      "LastLine": 4
      "VarList": 
        "Line": 4
        "Name": "a"
      "ExpList": 
        "Line": 4
        "Val": 1
    }{
      "LastLine": 5
      "VarList": 
        "Line": 5
        "Name": "b"
      "ExpList": 
        "Line": 5
        "Val": 2
    }{
      "LastLine": 6
      "VarList": 
        "Line": 6
        "Name": "c"
      "ExpList": 
        "Line": 6
        "Op": 17
        "exp1": 
          "Line": 6
          "Name": "a"
        "exp2": 
          "Line": 6
          "Name": "b"
    }{
      "LastLine": 7
      "VarList": 
        "Line": 7
        "Name": "d"
      "ExpList": 
        "Line": 7
        "Op": 23
        "exp1": 
          "Line": 7
          "Name": "a"
        "exp2": 
          "Line": 7
          "Name": "b"
    }]
    "RetExps": 
      nil
    
  • test if
  • if a < 0 then
        a = 1
    elseif a == 1 then
        a = 2
    elseif a == 2 then
        a = 3
    elseif a == 3 then
        a = 4
    else
        error("not support")
    end
    
    
    "LastLine": 23,
    "Stats": [{
      "Exps": 
        "Line": 13
        "Op": 28
        "exp1": 
          "Line": 13
          "Name": "a"
        "exp2": 
          "Line": 13
          "Val": 0
        
        "Line": 15
        "Op": 32
        "exp1": 
          "Line": 15
          "Name": "a"
        "exp2": 
          "Line": 15
          "Val": 1
        
        "Line": 17
        "Op": 32
        "exp1": 
          "Line": 17
          "Name": "a"
        "exp2": 
          "Line": 17
          "Val": 2
        
        "Line": 19
        "Op": 32
        "exp1": 
          "Line": 19
          "Name": "a"
        "exp2": 
          "Line": 19
          "Val": 3
        
        <lua_exp.TrueExp object at 0x7fbfd31562e8>
      "Blocks": 
        "LastLine": 14,
        "Stats": [{
          "LastLine": 14
          "VarList": 
            "Line": 14
            "Name": "a"
          "ExpList": 
            "Line": 14
            "Val": 1
        }]
        "RetExps": 
          nil
        "LastLine": 16,
        "Stats": [{
          "LastLine": 16
          "VarList": 
            "Line": 16
            "Name": "a"
          "ExpList": 
            "Line": 16
            "Val": 2
        }]
        "RetExps": 
          nil
        "LastLine": 18,
        "Stats": [{
          "LastLine": 18
          "VarList": 
            "Line": 18
            "Name": "a"
          "ExpList": 
            "Line": 18
            "Val": 3
        }]
        "RetExps": 
          nil
        "LastLine": 20,
        "Stats": [{
          "LastLine": 20
          "VarList": 
            "Line": 20
            "Name": "a"
          "ExpList": 
            "Line": 20
            "Val": 4
        }]
        "RetExps": 
          nil
        "LastLine": 22,
        "Stats": [{
          "Line": 22,
          "LastLine": 22,
          "PrefixExp": {
            "Line": 22
            "Name": "error"
          },
          "NameExp": None,
          "Args": [{
            "Line": 22
            "Str": "not support"
          }]
        }]
        "RetExps": 
          nil
    }]
    "RetExps": 
      nil
    
  • test while
  • while b > 0 do
        b = a - 1
    end
    
    
    "LastLine": 11,
    "Stats": [{
      "While": 
        "Exp": 
          "Line": 9
          "Op": 28
          "exp1": 
            "Line": 9
            "Name": "b"
          "exp2": 
            "Line": 9
            "Val": 0
        "Block": 
          "LastLine": 10,
          "Stats": [{
            "LastLine": 10
            "VarList": 
              "Line": 10
              "Name": "b"
            "ExpList": 
              "Line": 10
              "Op": 15
              "exp1": 
                "Line": 10
                "Name": "a"
              "exp2": 
                "Line": 10
                "Val": 1
          }]
          "RetExps": 
            nil
    }]
    "RetExps": 
      nil
    
  • test repeat
  • repeat
        a = a + 1
    until a >= 10
    
    "LastLine": 27,
    "Stats": [{
      "Repeat": 
        "Block": 
          "LastLine": 26,
          "Stats": [{
            "LastLine": 26
            "VarList": 
              "Line": 26
              "Name": "a"
            "ExpList": 
              "Line": 26
              "Op": 17
              "exp1": 
                "Line": 26
                "Name": "a"
              "exp2": 
                "Line": 26
                "Val": 1
          }]
          "RetExps": 
            nil
        "Exp": 
          "Line": 27
          "Op": 31
          "exp1": 
            "Line": 27
            "Name": "a"
          "exp2": 
            "Line": 27
            "Val": 10
    }]
    "RetExps": 
      nil
    
  • test for
  • for i = 1, 10, 3 do
        sum = sum + i
    end
    
    
    "LastLine": 31,
    "Stats": [{
      Line of for: 29
      Line of do: 29
      Var name: i
      Init exp: 
        "Line": 29
        "Val": 1
        
      Limit exp: 
        "Line": 29
        "Val": 10
        
      Step exp: 
        "Line": 29
        "Val": 3
        
      Block: 
        "LastLine": 30,
        "Stats": [{
          "LastLine": 30
          "VarList": 
            "Line": 30
            "Name": "sum"
          "ExpList": 
            "Line": 30
            "Op": 17
            "exp1": 
              "Line": 30
              "Name": "sum"
            "exp2": 
              "Line": 30
              "Name": "i"
        }]
        "RetExps": 
          nil
    }]
    "RetExps": 
      nil
    
    for a in table do
        sum = sum + a
    end
    
    
    "LastLine": 35,
    "Stats": [{
      Line of do: 33
        a  "Line": 33
        "Name": "table"
        "LastLine": 34,
        "Stats": [{
          "LastLine": 34
          "VarList": 
            "Line": 34
            "Name": "sum"
          "ExpList": 
            "Line": 34
            "Op": 17
            "exp1": 
              "Line": 34
              "Name": "sum"
            "exp2": 
              "Line": 34
              "Name": "a"
        }]
        "RetExps": 
          nil
    }]
    "RetExps": 
      nil
    
  • test function
  • function isok(a, n, c)
        return a + n == c
    end
    
    isok(1, 2, 3)
    
    
    "LastLine": 42,
    "Stats": [{
      "LastLine": 38
      "VarList": 
        Line: 38
        LastLine: 40
        ParList: 
          a
          n
          c
        IsVarArg: False
        Block:   "LastLine": 39,
          "Stats": []
          "RetExps": 
            "Line": 39
            "Op": 32
            "exp1": 
              "Line": 39
              "Op": 17
              "exp1": 
                "Line": 39
                "Name": "a"
              "exp2": 
                "Line": 39
                "Name": "n"
            "exp2": 
              "Line": 39
              "Name": "c"
      "ExpList": 
        "Line": 38
        "Name": "isok"
    }{
      "Line": 42,
      "LastLine": 42,
      "PrefixExp": {
        "Line": 42
        "Name": "isok"
      },
      "NameExp": None,
      "Args": [{
        "Line": 42
        "Val": 1
      }{
        "Line": 42
        "Val": 2
      }{
        "Line": 42
        "Val": 3
      }]
    }]
    "RetExps": 
      nil
    

    16.test optimizer
    a = true and false and true and false
    b = true or false or false
    
    a = 3 & 2
    b = 3 | 4
    c = 3 ^ 4
    d = 8 >> 2
    e = 8 << 2
    
    a = 3 + 4
    b = 5 - 3
    c = 3 * 4
    d = 10 / 3
    e = 10 // 3
    f = 10 % 3
    
    a = 3 ^ (1+2)
    
    a = -(-(-(-(-4))))
    
    a = ~4
    
    a = not true
    
    
    
    "LastLine": 89,
    "Stats": [{
      "LastLine": 67
      "VarList": 
        "Line": 67
        "Name": "a"
      "ExpList": 
        <lua_exp.FalseExp object at 0x7f66e29b2d30>
    }{
      "LastLine": 68
      "VarList": 
        "Line": 68
        "Name": "b"
      "ExpList": 
        <lua_exp.TrueExp object at 0x7f66e29b2cf8>
    }{
      "LastLine": 70
      "VarList": 
        "Line": 70
        "Name": "a"
      "ExpList": 
        "Line": 70
        "Val": 2
    }{
      "LastLine": 71
      "VarList": 
        "Line": 71
        "Name": "b"
      "ExpList": 
        "Line": 71
        "Val": 7
    }{
      "LastLine": 72
      "VarList": 
        "Line": 72
        "Name": "c"
      "ExpList": 
        "Line": 72
        "Val": 81.0
    }{
      "LastLine": 73
      "VarList": 
        "Line": 73
        "Name": "d"
      "ExpList": 
        "Line": 73
        "Val": 2
    }{
      "LastLine": 74
      "VarList": 
        "Line": 74
        "Name": "e"
      "ExpList": 
        "Line": 74
        "Val": 32
    }{
      "LastLine": 76
      "VarList": 
        "Line": 76
        "Name": "a"
      "ExpList": 
        "Line": 76
        "Val": 7
    }{
      "LastLine": 77
      "VarList": 
        "Line": 77
        "Name": "b"
      "ExpList": 
        "Line": 77
        "Val": 2
    }{
      "LastLine": 78
      "VarList": 
        "Line": 78
        "Name": "c"
      "ExpList": 
        "Line": 78
        "Val": 12
    }{
      "LastLine": 79
      "VarList": 
        "Line": 79
        "Name": "d"
      "ExpList": 
        "Line": 79
        "Val": 3.3333333333333335
    }{
      "LastLine": 80
      "VarList": 
        "Line": 80
        "Name": "e"
      "ExpList": 
        "Line": 80
        "Val": 3
    }{
      "LastLine": 81
      "VarList": 
        "Line": 81
        "Name": "f"
      "ExpList": 
        "Line": 81
        "Val": 1
    }{
      "LastLine": 83
      "VarList": 
        "Line": 83
        "Name": "a"
      "ExpList": 
        "Line": 83
        "Val": 27.0
    }{
      "LastLine": 85
      "VarList": 
        "Line": 85
        "Name": "a"
      "ExpList": 
        "Line": 85
        "Val": -4
    }{
      "LastLine": 87
      "VarList": 
        "Line": 87
        "Name": "a"
      "ExpList": 
        -5
    }{
      "LastLine": 89
      "VarList": 
        "Line": 89
        "Name": "a"
      "ExpList": 
        <lua_exp.FalseExp object at 0x7f66e29417b8>
    }]
    "RetExps": 
      nil
    

    17.test local
    local a, b, c = 1, 3, 4
    
    local function abc()
    end
    
    a = {1, 2, 3}
    
    
    "LastLine": 96,
    "Stats": [{
      <lua_stat.LocalVarDeclStat object at 0x7f4b91e7bd30>
    }{
      <lua_stat.LocalFuncDefStat object at 0x7f4b91e7be10>
    }{
      "LastLine": 96
      "VarList": 
        "Line": 96
        "Name": "a"
      "ExpList": 
        <lua_exp.TableConstructorExp object at 0x7f4b91e7bef0>
    }]
    "RetExps": 
      nil
    

    18.test var param
    function test(a, b, ...)
    end
    
    
    "LastLine": 99,
    "Stats": [{
      "LastLine": 98
      "VarList": 
        Line: 98
        LastLine: 99
        ParList: 
          a
          b
        IsVarArg: True
        Block:   "LastLine": 98,
          "Stats": []
          "RetExps": 
            nil
      "ExpList": 
        "Line": 98
        "Name": "test"
    }]
    "RetExps": 
      nil