爬虫入门（三）

分类: IT文章 • 2023-12-22 21:08:19

1.处理验证码

云打码平台
    注册：普通用户和开发用户
    登录：
        登录普通用户（查看余额）
        登录开发者用户：
            创建一个软件：我的软件->创建软件
            下载实例代码：开发者中心->下载最新的DLL->pythonHttp实例代码下载

import http.client, mimetypes, urllib, json, time, requests

######################################################################

class YDMHttp:

    apiurl = 'http://api.yundama.com/api.php'
    username = ''
    password = ''
    appid = ''
    appkey = ''

    def __init__(self, username, password, appid, appkey):
        self.username = username  
        self.password = password
        self.appid = str(appid)
        self.appkey = appkey

    def request(self, fields, files=[]):
        response = self.post_url(self.apiurl, fields, files)
        response = json.loads(response)
        return response
    
    def balance(self):
        data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['balance']
        else:
            return -9001
    
    def login(self):
        data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['uid']
        else:
            return -9001

    def upload(self, filename, codetype, timeout):
        data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)}
        file = {'file': filename}
        response = self.request(data, file)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['cid']
        else:
            return -9001

    def result(self, cid):
        data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)}
        response = self.request(data)
        return response and response['text'] or ''

    def decode(self, filename, codetype, timeout):
        cid = self.upload(filename, codetype, timeout)
        if (cid > 0):
            for i in range(0, timeout):
                result = self.result(cid)
                if (result != ''):
                    return cid, result
                else:
                    time.sleep(1)
            return -3003, ''
        else:
            return cid, ''

    def report(self, cid):
        data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'}
        response = self.request(data)
        if (response):
            return response['ret']
        else:
            return -9001

    def post_url(self, url, fields, files=[]):
        for key in files:
            files[key] = open(files[key], 'rb');
        res = requests.post(url, files=files, data=fields)
        return res.text

######################################################################

# 用户名 （普通用户）
username    = 'lianda0707'

# 密码
password    = 'ld18513185949.07'                            

# 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
appid       = 6998                                     

# 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
appkey      = 'cf4dda0913c13bb707a251a847bce5a1'    

# 图片文件
filename    = 'getimage.jpg'                        

# 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
codetype    = 1004

# 超时时间，秒
timeout     = 10                                    

# 检查
if (username == 'username'):
    print('请设置好相关参数再测试')
else:
    # 初始化
    yundama = YDMHttp(username, password, appid, appkey)

    # 登陆云打码
    uid = yundama.login();
    print('uid: %s' % uid)

    # 查询余额
    balance = yundama.balance();
    print('balance: %s' % balance)

    # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果
    cid, result = yundama.decode(filename, codetype, timeout);
    print('cid: %s, result: %s' % (cid, result))

######################################################################

DLL文件

def get_code(username, pwd, filepath, codetype):
    # 用户名 （普通用户）
    username    = username

    # 密码
    password    = pwd                            

    # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appid       = 6998                                     

    # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appkey      = 'cf4dda0913c13bb707a251a847bce5a1'    

    # 图片文件
    filename    = filepath                        

    # 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
    codetype    = codetype

    # 超时时间，秒
    timeout     = 20                                    
    
    result = None
    
    # 检查
    if (username == 'username'):
        print('请设置好相关参数再测试')
    else:
        # 初始化
        yundama = YDMHttp(username, password, appid, appkey)

        # 登陆云打码
        uid = yundama.login();
        print('uid: %s' % uid)

        # 查询余额
        balance = yundama.balance();
        print('balance: %s' % balance)

        # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果
        cid, result = yundama.decode(filename, codetype, timeout);
        print('cid: %s, result: %s' % (cid, result))
        
    return result
# 人人网的模拟登录
import requests
import urllib
from lxml import etree
# 获取session对象
session = requests.Session()
# 将验证码图片下载
url = 'http://www.renren.com/'
headers = {
#     'Connection': 'close',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36'
}

page_text = requests.get(url=url, headers=headers).text
tree = etree.HTML(page_text)
code_img_url = tree.xpath('//*[@>)[0]
urllib.request.urlretrieve(url=code_img_url, filename='code.jpg')

# 识别验证码图片中的数据值
code_data = get_code('lianda0707', 'ld18513185949.07', './code.jpg', 2004)

# 模拟登录
login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2019141628905 '
data = {
    "email":"15764551436",
    "icode":code_data,
    "origURL":"http://www.renren.com/home",
    "domain":"renren.com",
    "key_id":"1",
    "captcha_type":"web_login",
    "password":"96afa5f31bb9fb28ecc4c5eeda29a68b5aac63e38c3d51360ddba0b0f63e3238",
    "rkey":"5f7a1d96ff48551fe2910f08a1a2ddcd",
    "f":"",
}
#该次请求产生的cookie会被自动存储到session对象中
ll = session.post(url=login_url,data=data,headers=headers).text
print(ll)
url = 'http://www.renren.com/969892564/profile'
page_text = session.get(url=url,headers=headers).text

with open('renren.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

# 模拟登录古诗文忘
s = requests.Session()
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
login_url = 'https://so.gushiwen.org/user/login.aspx?from='
page_text = s.get(url=login_url, headers=headers).text
tree = etree.HTML(page_text)
img_src = 'https://so.gushiwen.org' + tree.xpath('//*[@>)[0]
img_data = s.get(url=img_src, headers=headers).content

with open('./img.jpg','wb') as fp:
    fp.write(img_data)

img_text = get_code('lianda0707', 'ld18513185949.07', './img.jpg', 1004)

# 模拟登录
url = 'https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx'
data = {
    "__VIEWSTATE":"FtrD1RIDtJDgCDOLII4PoI3KO8h6/WZp+uZrKxMZEhiTioqiAR9+e0f3i8Mvg+jCiSjYf9Et4rFcPgmghFuf7xRaS6K9nsJS4HFBCRKcUvPovmSc9fyYoTjUGZY=",
    "__VIEWSTATEGENERATOR":"C93BE1AE",
    "from":"http://so.gushiwen.org/user/collect.aspx",
    "email":"849784670@qq.com",
    "pwd":"ld18513185949.07",
    "code":img_text,
    "denglu":"登录",
}

page_text = s.post(url=url, headers=headers, data=data).text
print(page_text)
with open('./gushiwen.html','w',encoding='utf8') as fp:
    fp.write(page_text)

2.selenium的使用

环境安装： pip install selenium
编码流程：
    导包：from selenium import webdriver
    实例化某一款浏览器对象
    自制定自动化操作代码

from selenium import webdriver
from time import sleep

bro = webdriver.Chrome(executable_path=r'F:爬虫+数据day03chromedriver_win32chromedriver.exe')
bro.get(url='https://www.baidu.com/')
sleep(2)
text_input = bro.find_element_by_id('kw')

text_input.send_keys("人民币")
sleep(2)
bro.find_element_by_id('su').click()
sleep(3)

# 获取当前页面源码数据
print(bro.page_source)
bro.quit()

 1 url = 'https://movie.douban.com/typerank?type_name=%E6%83%8A%E6%82%9A&type=19&interval_id=100:90&action='
 2 
 3 bro = webdriver.Chrome(executable_path=r'F:爬虫+数据day03chromedriver_win32chromedriver.exe')
 4 bro.get(url)
 5 sleep(3)
 6 
 7 bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
 8 sleep(3)
 9 bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
10 sleep(3)
11 bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
12 
13 sleep(2)
14 page_text = bro.page_source
15 
16 with open('./douban.html','w',encoding='utf8') as fp:
17     fp.write(page_text)
18 
19 sleep(1)
20 bro.quit()

# 获取豆瓣电影中更多电影详情数据

# 谷歌无头浏览器
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

#获取豆瓣电影中更多电影详情数据
url = 'https://movie.douban.com/typerank?type_name=%E6%83%8A%E6%82%9A&type=19&interval_id=100:90&action='
bro = webdriver.Chrome(executable_path=r'F:爬虫+数据day03chromedriver_win32chromedriver.exe',chrome_options=chrome_options) #多加一个参数
bro.get(url)
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(2)
page_text = bro.page_source

with open('./douban.html','w',encoding='utf-8') as fp:
    fp.write(page_text)
print(page_text)
sleep(1)

谷歌无头浏览器（同上）

爬虫入门（三）

1.处理验证码

2.selenium的使用

3.线程的使用

相关推荐