#coding=utf-8
import sys
with open('ch_input_gbk', 'r') as f1, open('ch_input_utf', 'r') as f2:
 for l1 in f1:
 lines = l1.strip().split('\t') # lines是list, 通过打印它可以看看str不同编码的内容
 sent = lines[0]   # sent是ch-str
 print lines, sent
 for l2 in f2:
 lines = l2.strip().split('\t')
 sent = lines[0]
 print lines, sent
 print sent.decode('utf8').encode('gbk')
 #print str(sent).decode('string_escape').decode('utf8').encode('gbk')

['\xd3\xc4\xc8\xcb\xd6\xf1\xc9\xa3\xd4\xb0'] 幽人竹桑园
['\xb9\xe9\xce\xd4\xbc\xc5\xce\xde\xd0\xfa'] 归卧寂无喧
['\xce\xef\xc7\xe9\xbd\xf1\xd2\xd1\xbc\xfb'] 物情今已见
['\xb4\xd3\xb4\xcb\xd3\xfb\xce\xde\xd1\xd4'] 从此欲无言
['\xe5\xb9\xbd\xe4\xba\xba\xe7\xab\xb9\xe6\xa1\x91\xe5\x9b\xad'] 骞戒汉绔规鍥
幽人竹桑园
['\xe5\xbd\x92\xe5\x8d\xa7\xe5\xaf\x82\xe6\x97\xa0\xe5\x96\xa7'] 褰掑崸瀵傛棤鍠
归卧寂无喧
['\xe7\x89\xa9\xe6\x83\x85\xe4\xbb\x8a\xe5\xb7\xb2\xe8\xa7\x81'] 鐗╂儏浠婂凡瑙
物情今已见
['\xe4\xbb\x8e\xe6\xad\xa4\xe6\xac\xb2\xe6\x97\xa0\xe8\xa8\x80'] 浠庢娆叉棤瑷
从此欲无言

reload(sys) 
sys.setdefaultencoding('utf8')

# -*- coding: utf-8 -*- 
import re 
def findPart(regex, text, name): 
 res = re.findall(regex, text) 
 print "There are %d %s parts:" % (len(res), name) 
 for r in res: 
 print r.encode('gbk')
 
sample = '''en: Regular expression is a powerful tool for manipulating text. 
  zh: 正则表达式是一种很有用的处理文本的工具。 
  jp: 正規表現は非常に役に立つツールテキストを操作することです。 
  jp-char: あアいイうウえエおオ 
  kr:"[\u4e00-\u9fa5]+", usample, "unicode chinese") 
findPart(u"[\uac00-\ud7ff]+", usample, "unicode korean") 
findPart(u"[\u30a0-\u30ff]+", usample, "unicode japanese katakana") 
findPart(u"[\u3040-\u309f]+", usample, "unicode japanese hiragana") 
findPart(u"[\u3000-\u303f\ufb00-\ufffd]+", usample, "unicode cjk Punctuation") 

s1 = '天天天向上天天向上' 
print (re.sub(ur'[\u4e00-\u9fa5]{1,}', u'1', s1.decode('utf8'))).encode('gbk') # 1
print (re.sub(ur'([\u4e00-\u9fa5])\1{1,}', u'1', s1.decode('utf8'))).encode('gbk') # 1向上1向上
print (re.sub(ur'([\u4e00-\u9fa5])\1{2,}', u'1', s1.decode('utf8'))).encode('gbk') # 1向上天天向上
s2 = '【aa】天天bb@cc'
print (re.sub(ur'【.*】', u'1', s2.decode('utf8'))).encode('gbk')   # 1天天bb@cc
print (re.sub(ur'@', u'1', s2.decode('utf8'))).encode('gbk')   # 【aa】天天bb1cc