1. 语料库准备
???? （需要的私聊博主）
2.处理语料文档
2.1处理多余字符
def string_process(x): #处理字符串
a=re.sub(r'\d{8}-\d{2}-\d{3}-\d{3}/m|[/a-z！。”“，、——\[\]（）：《》……A-Z？]', "", x)
#前面时间的正则表达式,后面部分删掉字母和其他符号
b=a.replace(" "," ")
return b.rstrip()
2.2 处理字典数据集
def file_process():
s=""
with open('199801-train.txt','r',encoding='UTF-8') as f:
for line1 in f:
a=string_process(line1)
a=string_process(a)
s+=a+"\n"
f.close()
with open("process_1.txt",'w',encoding='UTF-8') as w:
w.write(s)
w.close()
2.3处理测试数据集
def test_file_process():
s = ""
with open('199801-test.txt', 'r', encoding='UTF-8') as f:
for line1 in f:
a = string_process(line1)
a = string_process(a)
s += a + "\n"
f.close()
with open("test_process_1.txt", 'w', encoding='UTF-8') as w:
w.write(s)
w.close()
2.4处理测试集空格
def delete_space():
f = open('test_process_1.txt','r', encoding='utf-8')
f2 = open('test_endfile.txt', 'w', encoding='utf-8')
s = f.read()
r = ''.join(s.split())
f2.write(r)
3.FMM
training_words = 'result.txt'
def get_dic(training_words):
with open(training_words,'r',encoding='utf-8') as f:
try:
file_content = f.read().split()
finally:
f.close()
chars = list(set(file_content))
return chars
dic = get_dic(training_words)
def readfile():
max_length=0
for i in dic:
max_length=max(max_length,len(i))##获得最大长度
zz=max_length
f=open("test_endfile.txt",'r',encoding='utf-8')
ff=open("生成文档正向最大匹配(真).txt",'w',encoding='utf-8')
lines=f.readlines()
f.close()
for line in lines:#分别对每一行进行正向最大匹配处理
max_length=zz
my_list=[]
len_hang=len(line)
while len_hang>0:
tryW=line[0:max_length]##切割字符串
while tryW not in dic:
if len(tryW)==1:#长度为1的时候就直接退出
break;
tryW=tryW[0:len(tryW)-1]
my_list.append(tryW)
line=line[len(tryW):]
len_hang=len(line)
for i in my_list:
ff.write(i+"/")
ff.close()
print("complete !!!")
readfile()
4.BMM
training_words = 'result.txt'
def get_dic(training_words):
with open(training_words,'r',encoding='utf-8') as f:
try:
file_content = f.read().split()
finally:
f.close()
chars = list(set(file_content))
return chars
dic = get_dic(training_words)
def readfile():
max_length=0
for i in dic:
max_length=max(max_length,len(i))##获得最大长度
zz=max_length
f=open("测试.txt",'r',encoding='utf-8')
ff=open("生成文档逆向最大匹配.txt",'w',encoding='utf-8')
lines=f.readlines()
f.close()
for line in lines:#分别对每一行进行逆向最大匹配处理
max_length=zz
my_list = []
len_hang=len(line)
while len_hang>0:
tryW=line[max(0,len_hang-max_length):len_hang]#防止溢出
while tryW not in dic:
if len(tryW)==1:
break;
tryW=tryW[1:len(tryW)]#这里注意，一直是从1开始的
my_list.append(tryW)
line=line[0:len_hang-len(tryW)]
len_hang=len(line)
while len(my_list):
tt=my_list.pop()#这里类似栈的操作
ff.write(tt+"/")
ff.close()
readfile()##012345
|