python爬虫输入标题百度百科获取内容
##原始诉求，经过标题获取内容翻译英文再翻译中文，提高原创度
import requests
import re
from lxml import etree
from translate import Translator
import urllib.request
import urllib.parse
import json
def first_url(url):
send_headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36",
"Connection": "keep-alive",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.8"
}
res = requests.get(url,send_headers)
res.close()
res.encoding= 'utf-8'
html=res.text
return html
#获取第一个百度经验内容下p标签的文本内容
def get_result(html):
title=[]
datahtml=etree.HTML(html)
html_data=datahtml.xpath('/html/body/section/div/div/article/div/div/div/ol/li/div/p')
print(type(html_data))
finalre=''
for i in html_data:
title.append(i.text)
#print(i.text)
finalre=''.join(title)
#print(finalre)
f=open('f.txt','w',encoding='utf-8')
f.write(finalre)
f.close()
#中文转英文
def youdao_translate(content):
'''实现有道翻译的接口'''
youdao_url = 'http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule'
data = {}
data['i']= content
data['from'] = 'AUTO'
data['to'] = 'AUTO'
data['smartresult'] = 'dict'
data['client'] = 'fanyideskweb'
data['salt'] = '1525141473246'
data['sign'] = '47ee728a4465ef98ac06510bf67f3023'
data['doctype'] = 'json'
data['version'] = '2.1'
data['keyfrom'] = 'fanyi.web'
data['action'] = 'FY_BY_CLICKBUTTION'
data['typoResult'] = 'false'
data = urllib.parse.urlencode(data).encode('utf-8')
youdao_response = urllib.request.urlopen(youdao_url, data)
youdao_html = youdao_response.read().decode('utf-8')
target = json.loads(youdao_html)
trans = target['translateResult']
ret = ''
for i in range(len(trans)):
line = ''
for j in range(len(trans[i])):
line = trans[i][j]['tgt']
ret += line + '
'
#print(ret)
return ret
if __name__ == "__main__":
searce_value='为什么流鼻血'
url="http://jingyan.baidu.com/search?word=%s"%searce_value
first_html_url=first_url(url) #首次获取url结果内容
second_url=eval(re.findall(r'"/article/.*.html"',first_html_url)[0]) #选择第一个百度经验链接
second_resul_url="http://jingyan.baidu.com%s"%second_url
se_html_resu=first_url(second_resul_url) #第一个百度经验链接获取内容
get_result(se_html_resu) #进行获取p标签的内容打印输出
fif=open('f.txt','r',encoding='utf-8')
linelist=fif.read().split('。')
#print(linelist)
for line in linelist:
if line.find('？')>0:
#print('-----',line)
linli=line.split('？')
for l in linli:
if len(l) > 0:
fi=youdao_translate(l)
zhfi=youdao_translate(fi)
print(zhfi)
if len(line) > 0:
print(line)
fi=youdao_translate(line)
zhfi=youdao_translate(fi)
print(zhfi)