hoi4 lex

import re, sys

testfile = "test.save.txt"

with open(testfile,'r') as fd:
    
    chars = fd.read()

def lex(chars, token_expr_list):

    pos=0
    tokens=[]
    while pos < len(chars):
        match=None
        for token_expr in token_expr_list:
            pattern, tag = token_expr
            r = re.compile(pattern)
            match = r.match(chars,pos)
            if match:
                text = match.group(0)
                if tag:
                    token = (text, tag)
                    tokens.append(token)
                    print("token : {}".format(token))
                break
            
        if not match:
            print("error with : {}".format(pos))
            sys.exit(1)
        
        else:
            pos = match.end(0)

    return tokens



# for line in all_lines:
#     print(line)    


token_expr_list = [
    (r'[ \n\t]+', None),
    (r'=[ \n\t]*{',"ITEM_START"),
    (r'}',"ITEM_END"),
    (r'[0-9\.]+','NUM'),
    (r'[a-zA-Z0-9_]+','STRING')

]

tokens = lex(chars, token_expr_list)

print(tokens)


    


print("finish")