Python爬虫学习1: Requests模块的使用

> pip install requests
Collecting requests
  Downloading requests-2.11.1-py2.py3-none-any.whl <514kB>
Installing collected packages: requests
Successfully installed requests-2.11.1

>>> import requests
 
# 请求豆瓣读书网站点
>>> r = requests.get('https://book.douban.com/') # 返回一个实例,包含了很多的信息
>>> r.status_code  # 响应状态码,200表示服务器已成功处理了请求
200
>>> r.headers['content-type'] # 浏览器根据该参数决定对文档进行如何解析,text是文档类型,html则是文档子类型;
'text/html; charset=utf-8'
>>> r.encoding  # 所请求网页的编码方式
'utf-8'
>>> r.text  # 所请求网页的内容
...
>>> r.cookie   # 网页的cookie内容
<RequestsCookieJar[Cookie(version=0, name='bid', value='mGadkt5VXtk', port=None, port_specified=False, domain='.douban.com', domain_specified=True, domain_initial_dot=True, path='/', path_specified=True, secure=False, expires=1508551773, discard=False, comment=None, comment_url=None, rest={}, rfc2109=False)]>
>>> r.headers  # 网页的头
{'X-Xss-Protection': '1; mode=block', 'X-DAE-App': 'book', 'X-Content-Type-Options': 'nosniff', 'Content-Encoding': 'gzip', 'Transfer-Encoding': 'chunked', 'Set-Cookie': 'bid=mGadkt5VXtk; Expires=Sat, 21-Oct-17 02:09:33 GMT; Domain=.douban.com; Path=/', 'Expires': 'Sun, 1 Jan 2006 01:00:00 GMT', 'Vary': 'Accept-Encoding', 'Keep-Alive': 'timeout=30', 'X-DAE-Node': 'dis16', 'X-DOUBAN-NEWBID': 'mGadkt5VXtk', 'X-Douban-Mobileapp': '0', 'Connection': 'keep-alive', 'Pragma': 'no-cache', 'Cache-Control': 'must-revalidate, no-cache, private', 'Date': 'Fri, 21 Oct 2016 02:09:33 GMT', 'Strict-Transport-Security': 'max-age=15552000;', 'Server': 'dae', 'Content-Type': 'text/html; charset=utf-8'}
>>> r.url      # 实际的网址
u'https://book.douban.com/'

1 r = requests.post('https://book.douban.com/')
2 r = requests.put('https://book.douban.com/')
3 r = requests.delete('https://book.douban.com/')
4 r = requests.head('https://book.douban.com/')
5 r = requests.options('https://book.douban.com/')

>>> param = {'key1':'value1', 'key2':'value2'}  # 先生成一个字典
>>> param
{'key2': 'value2', 'key1': 'value1'}
>>> r = requests.get('http://httpbin.org/get', params = param)
>>> r.url
u'http://httpbin.org/get?key2=value2&key1=value1'  # 可以看到,参数之间用&隔开,参数名和参数值之间用=隔开

# 还可以将一个列表作为参数值传入
>>> param = {'key1': 'value1', 'key2': ['value2', 'value3']}  
>>> param
{'key2': ['value2', 'value3'], 'key1': 'value1'}
>>> r = requests.get('http://httpbin.org/get', params=param)
>>> r.url
u'http://httpbin.org/get?key2=value2&key2=value3&key1=value1'

>>> r = requests.get('https://www.douban.com/')
>>> r.status_code
200>>> r.encoding
'utf-8'

>>> r.encoding
'utf-8'
>>> r.encoding = 'ISO-8859-1'

from PIL import Image
from io import BytesIO

r = requests.get('http://pic6.huitu.com/res/20130116/84481_20130116142820494200_1.jpg')
i = Image.open(BytesIO(r.content)) # 得到图像
i.save('sample.jpg', 'jpeg')       # 保存图像

>>> r = requests.get('https://github.com/timeline.json')
>>> r.json()
{u'documentation_url': u'https://developer.github.com/v3/activity/events/#list-public-events', u'message': u'Hello there, wayfaring stranger. If youu2019re reading this then you probably didnu2019t see our blog post a couple of years back announcing that this API would go away: http://git.io/17AROg Fear not, you should be able to get what you need from the shiny new Events API instead.'}
>>> r.json
<bound method Response.json of <Response [410]>>