テキストプリプロセッシングノット
4906 ワード
#coding=gbk
'''
Created on 2016-01-14 08:56:27
@author: heng
'''
import re
import datetime
start = datetime.datetime.now()
file1 = open('I:\\test1\\8. Tweet and Retweets with indexed content\\weibocontents\\Retweet_content.txt','r')
file2 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test_final1.txt','w')
rule3 = '^@.*(
)$'
rule2 = '^(retweet).*
'
rule1 = '^(link).*(
)$'
compile_rule1 = re.compile(rule1)
compile_rule2 = re.compile(rule2)
compile_rule3 = re.compile(rule3)
for i in file1:
i1 = ''.join(i)
i2 = compile_rule1.sub('',i1)
i3 = compile_rule2.sub('',i2)
i4 = compile_rule3.sub('',i3)
file2.write(str(i4))
file1.close()
file2.close()
end = datetime.datetime.now()
print end-start
print 'program is ended!!!!!!!!'
#coding=gbk
'''
Created on 2016-01-14 15:08:49
@author: heng
'''
import datetime
def retweet_content(file1,file2):
j = 0;k=0
for i in file1:
j = j+1
i1 = ''.join(i)
i2 = i1.strip('
').split('\t')
if len(i2)==4:
str_0 = ''.join(i2[0])
if len(i2)==3:
file2.write('\t'.join(i2)+'\t'+str_0)
k = j
if j==k+1:
file2.write('\t'+str(i1))
file1.close()
file2.close()
def time_processing(file3,file4):
for i in file3:
i1=''.join(i)
i2 = i1.split('\t')
i3 = ''.join(i2[1])
i3 = i3[0:10]+' '+i3[11:19]
i2[1] = i3
i4 = '\t'.join(i2)
file4.write(str(i4))
file3.close()
file4.close()
start = datetime.datetime.now()
file1 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test_final1.txt','r')
file2 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test3.txt','w')
retweet_content(file1, file2)
file3 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test3.txt','r')
file4 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test4.txt','w')
time_processing(file3,file4)
end = datetime.datetime.now()
print end-start
print 'program is ended!!!!!!!!'