python实现词法分析

#请先安装Ply

# -*- coding: utf-8 -*-
#--------------------------------------------------------------------------
#Author:Jmdebugger
#email: pengkailb@gmail.com
#date: 2013-9-17
#--------------------------------------------------------------------------
import ply.lex as lex

tokens = [
    "TOKEN_IDENT",
    "TOKEN_INT",
    "TOKEN_FLOAT",
    "TOKEN_STRING",
    "TOKEN_OP",
    "TOKEN_DELIM_COMMA",        #,
    "TOKEN_DELIM_OPEN_PAREN",   #(
    "TOKEN_DELIM_CLOSE_PAREN",  #)
    "TOKEN_DELIM_OPEN_BRACKET", #[
    "TOKEN_DELIM_CLOSE_BRACKET",#]
    "TOKEN_DELIM_OPEN_BRACE",   #{
    "TOKEN_DELIM_CLOSE_BRACE",  #}
    "TOKEN_DELIM_SEMICOLON"    #;

    ]

reserved = {
   'if'     :   'TOKEN_RSRVD_IF',
   'else'   :   'TOKEN_RSRVD_ELSE',
   'true'   :   'TOKEN_RSRVD_TRUE',
   'false'  :   'TOKEN_RSRVD_FALSE',
   'while'  :   'TOKEN_RSRVD_WHILE',
   'break'  :   'TOKEN_RSRVD_BREAK',
   'continue':  'TOKEN_RSRVD_CONTINUE',
   'goto'   :   'TOKEN_RSRVD_GOTO',
   'func'   :   'TOKEN_RSRVD_FUNC',
   'var'    :   'TOKEN_RSRVD_VAR',
   'for'    :   'TOKEN_RSRVD_FOR',
   'return' :   'TOKEN_RSRVD_RETURN'
}

tokens += reserved .values()

t_ignore = r' \t\r'
def t_COMMENT(t):
    r'(/\*(.|\n)*?\*/)|(\/\/.*)'
    pass

def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

def t_error(t):
    print "LaunchScript error: "+repr(t.value)

def t_TOKEN_IDENT(t):
    r'[a-zA-Z_][a-zA-Z_0-9]*' #标识符
    t.type = reserved.get(t.value , 'TOKEN_IDENT')
    return t

def t_TOKEN_INT(t):
    r'(0x[a-fA-F0-9]+)|([0-9]+)'
    return t

t_TOKEN_FLOAT =  r'[0-9]*\.[0-9]+'
t_TOKEN_STRING = r'(\"([^\\\r]|(\\.))*?\")' #|(\"([^\\\n]|(\\.))*?\")' only for windows
t_TOKEN_DELIM_COMMA = r'\,'
t_TOKEN_DELIM_OPEN_PAREN = r'\('
t_TOKEN_DELIM_CLOSE_PAREN = r'\)'
t_TOKEN_DELIM_OPEN_BRACKET = r'\['
t_TOKEN_DELIM_CLOSE_BRACKET = r'\]'
t_TOKEN_DELIM_OPEN_BRACE = r'\{'
t_TOKEN_DELIM_CLOSE_BRACE = r'\}'
t_TOKEN_DELIM_SEMICOLON = r'\;'

def t_TOKEN_OP(t):
    r'(\<\<\=)|(\>\>\=)|([\+\-\*\/\%\&\|\^\=\!\>\<]\=)|(\|\|)|(\&\&)|(\+\+)|(\-\-)|[\+\-\*\/\%\^\=\&\|\>\<\!\~]'
    return t

if __name__ == "__main__":
    lexer = lex.lex()
    f = open("./test.txt" , 'rb')
    data = f.read()
    f.close()
    lexer.input(data)

    while True:
        tok = lexer.token()
        if not tok: break      # No more input
        print tok.value+"\t---->\t"+tok.type