import re, sys testfile = "test.save.txt" with open(testfile,'r') as fd: chars = fd.read() def lex(chars, token_expr_list): pos=0 tokens=[] while pos < len(chars): match=None for token_expr in token_expr_list: pattern, tag = token_expr r = re.compile(pattern) match = r.match(chars,pos) if match: text = match.group(0) if tag: token = (text, tag) tokens.append(token) print("token : {}".format(token)) break if not match: print("error with : {}".format(pos)) sys.exit(1) else: pos = match.end(0) return tokens # for line in all_lines: # print(line) token_expr_list = [ (r'[ \n\t]+', None), (r'=[ \n\t]*{',"ITEM_START"), (r'}',"ITEM_END"), (r'[0-9\.]+','NUM'), (r'[a-zA-Z0-9_]+','STRING') ] tokens = lex(chars, token_expr_list) print(tokens) print("finish")
Categories: Uncategorized
0 Comments