pythonテキスト処理(単語を結合して記号を除去する)
7936 ワード
import re
import jieba.analyse
import codecs
import pandas as pd
def simplification_text(xianbingshi):
""" """
xianbingshi_simplification = []
with codecs.open(xianbingshi,'r','utf8') as f:
for line in f :
line = line.strip()
line_write = re.findall('(?<=\).*?(?=\) ',line)
for line in line_write:
xianbingshi_simplification.append(line)
with codecs.open(r'C:\Users\Administrator.SC-201812211013\PycharmProjects\untitled29\yiwoqu\code\xianbingshi_write.txt','w','utf8') as f:
for line in xianbingshi_simplification:
f.write(line + '
')
def jieba_text():
""""""
word_list = []
data = open(r"C:\Users\Administrator.SC-201812211013\PycharmProjects\untitled29\xianbingshi_write.txt", encoding='utf-8').read()
seg_list = jieba.cut(data, cut_all=False) #
for i in seg_list:
word_list.append(i.strip())
data_quchong = pd.DataFrame({'a':word_list})
data_quchong.drop_duplicates(subset=['a'],keep='first',inplace=True)
word_list = data_quchong['a'].tolist()
with codecs.open('word.txt','w','utf8')as w:
for line in word_list:
w.write(line + '
')
def word_messy(word):
""" """
word_sub_list = []
with codecs.open(word,'r','utf8') as f:
for line in f:
line_sub = re.sub("^[1-9]\d*\.\d*|^[A-Za-z0-9]+$|^[0-9]*$|^(-?\d+)(\.\d+)?$|^[A-Za-z0-9]{4,40}.*?",'',line)
word_sub_list.append(line_sub)
word_sub_list.sort()
with codecs.open('word.txt','w','utf8')as w:
for line in word_sub_list:
w.write(line.strip("
") + '
')
if __name__ == '__main__':
xianbingshi = r'C:\Users\Administrator.SC-201812211013\PycharmProjects\untitled29\yiwoqu\xianbingshi_sub_sen_all(1).txt'
# simplification_text(xianbingshi)
# word = r'C:\Users\Administrator.SC-201812211013\PycharmProjects\untitled29\word.txt'
simplification_text(xianbingshi)
転載先:https://www.cnblogs.com/yiwoqu/p/11542002.html