1. process_item(item, spider)。        # 核心方法
    process_item()是必须要实现的方法，它必须返回Item类型的值或者抛出一个DropItem异常。
        返回的是Item对象，那么此Item会被低优先级的Item Pipeline的process_item()方法处理，直到所有的方法被调用完毕。
        如果它抛出的是DropItem异常，那么此Item会被丢弃，不再进行处理。
        
    process_item()方法的参数有如下两个。
        item，是Item对象，即被处理的Item。
        spider，是Spider对象，即生成该Item的Spider。
    
2. open_spider(spider)。
    在Spider开启的时候被自动调用的。在这里我们可以做一些初始化操作，如开启数据库连接等。其中，参数spider就是被开启的Spider对象。

3. close_spider(spider)。
    在Spider关闭的时候自动调用的。在这里我们可以做一些收尾工作，如关闭数据库连接等。其中，参数spider就是被关闭的Spider对象。

4. from_crawler(cls, crawler)。
    是一个类方法，用@classmethod标识，是一种依赖注入的方式。它的参数是crawler，通过crawler对象，我们可以拿到Scrapy的所有核心组件，如全局配置的每个信息，然后创建一个Pipeline实例。
    

import scrapy

class TxtItem(scrapy.Item):
    content = scrapy.Field()
    href = scrapy.Field()

from scrapy.exceptions import DropItem

class TxtPipeline(object):
    """
    源码内容：
        1. 判断当前XdbPipeline类中是否有from_crawler
            有：
                obj = XdbPipeline.from_crawler(....)
            否：
                obj = XdbPipeline()
        2. obj.open_spider()

        3. obj.process_item()/obj.process_item()/obj.process_item()/obj.process_item()/obj.process_item()

        4. obj.close_spider()
    """

    def __init__(self, path):
        self.f = None
        self.path = path

    @classmethod
    def from_crawler(cls, crawler):
        # 初始化时, 用于创建pipeline对象
        path = crawler.settings.get('TXTPATH')
        return cls(path)

    def open_spider(self, spider):
        # 爬虫开始执行
        print('爬虫开始了')
        self.f = open(self.path, 'a+', encoding='utf-8')

    def process_item(self, item, spider):
        # item为字典数据格式
        # yield item对象时, 执行
        print('保存在到数据库中...')
        self.f.write(json.dumps(dict(item), ensure_ascii=False)+'
')
        # return item        # 交给下一个pipeline的process_item()方法
        raise DropItem()    # 下一个pipeline的process_item方法中断不执行

    def close_spider(self, spider):
        # 爬虫结束时执行
        print('爬虫结束了')
        self.f.close()

ITEM_PIPELINES = {
   'scrapy_env.pipelines.TxtPipeline': 300,
}

yield TxtItem(content=content, href=href)    # 会自动执行pipeline里的process_item方法

from scrapy.dupefilter import BaseDupeFilter
from scrapy.utils.request import request_fingerprint

class CustomizeDupeFilter(BaseDupeFilter):

    """
    自定义的url去重规则
    :return: True表示已经访问过；False表示未访问过
    """

    def __init__(self):
        self.visited_fd = set()

    def request_seen(self, request):
        print('去重中')
        fd = request_fingerprint(request)       # url的唯一标示
        if fd in self.visited_fd:
            return True
        self.visited_fd.add(fd)

    def open(self):
        print('引擎开始///')

    def close(self, reason):
        print('引擎结束///')

import scrapy
from scrapy.http.request import Request
from scrapy.http.cookies import CookieJar

class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    allowed_domains = ['chouti.com']
    start_urls = ['http://chouti.com/']

    cookie_dict = {}

    def parse(self, response):

        # 6.1 去响应头中获取cookie, 保存在cookie_jar对象中
        cookie_jar = CookieJar()
        cookie_jar.extract_cookies(response, response.request)

        # 6.2 去cookie_jar对象中将cookie解析到字典中
        for k, v in cookie_jar._cookies.items():
            for i, j in v.items():
                for m, n in j.items():
                    self.cookie_dict[m] = n.value

        print(self.cookie_dict)

        # 6.3 携带cookie去登录
        yield Request(url='https://dig.chouti.com/login',
                      method='post',
                      headers={
                          'content-type': 'application/x-www-form-urlencoded; charset=UTF-8'
                      },
                      body='phone=8618648957737&password=/***/&oneMonth=1',
                      cookies=self.cookie_dict,
                      callback=self.check_result
                      )
                      
    def check_result(self, response):
        print(response.text)

import scrapy
from scrapy.http.request import Request

class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    allowed_domains = ['chouti.com']
    start_urls = ['http://chouti.com/']
    
    def start_requests(self):
        url = 'http://dig.chouti.com/'
        # 1 设置了meta的cookiejar=True
        yield Request(url=url, callback=self.login, meta={'cookiejar': True})

    def login(self, response):
        # print(response.headers.getlist('Set-Cookie'))
        yield Request(
            url='http://dig.chouti.com/login',
            method='POST',
            headers={'content-type': 'application/x-www-form-urlencoded; charset=UTF-8'},
            body='phone=8618648957737&password=woshiniba&oneMonth=1',
            callback=self.check_result,
            meta={'cookiejar': True}        # 2 会自动携带cookie过去
        )

    def check_result(self, response):
        print('检查结果')
        print(response.text)

import base64
import random
from six.moves.urllib.parse import unquote
try:
    from urllib2 import _parse_proxy
except ImportError:
    from urllib.request import _parse_proxy
from six.moves.urllib.parse import urlunparse
from scrapy.utils.python import to_bytes

class XdbProxyMiddleware(object):

    def _basic_auth_header(self, username, password):
        user_pass = to_bytes(
            '%s:%s' % (unquote(username), unquote(password)),
            encoding='latin-1')
        return base64.b64encode(user_pass).strip()

    def process_request(self, request, spider):
        PROXIES = [
            "http://root:woshiniba@192.168.11.11:9999/",
            "http://root:woshiniba@192.168.11.12:9999/",
            "http://root:woshiniba@192.168.11.13:9999/",
            "http://root:woshiniba@192.168.11.14:9999/",
            "http://root:woshiniba@192.168.11.15:9999/",
            "http://root:woshiniba@192.168.11.16:9999/",
        ]
        url = random.choice(PROXIES)

        orig_type = ""
        proxy_type, user, password, hostport = _parse_proxy(url)
        proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))

        if user:
            creds = self._basic_auth_header(user, password)
        else:
            creds = None
        request.meta['proxy'] = proxy_url
        if creds:
            request.headers['Proxy-Authorization'] = b'Basic ' + creds

class SpiderMiddleware(object):

    def process_spider_input(self,response, spider):
        """
        下载完成，执行，然后交给parse处理
        :param response: 
        :param spider: 
        :return: 
        """
        pass

    def process_spider_output(self,response, result, spider):
        """
        spider处理完成，返回时调用
        :param response:
        :param result:
        :param spider:
        :return: 必须返回包含 Request 或 Item 对象的可迭代对象(iterable)
        """
        return result

    def process_spider_exception(self,response, exception, spider):
        """
        异常调用
        :param response:
        :param exception:
        :param spider:
        :return: None,继续交给后续中间件处理异常；含 Response 或 Item 的可迭代对象(iterable)，交给调度器或pipeline
        """
        return None


    def process_start_requests(self,start_requests, spider):
        """
        爬虫启动时调用
        :param start_requests:
        :param spider:
        :return: 包含 Request 对象的可迭代对象
        """
        return start_requests

class DownMiddleware1(object):
    def process_request(self, request, spider):
        """
        请求需要被下载时，经过所有下载器中间件的process_request调用
        :param request: 
        :param spider: 
        :return:  
            None,继续后续中间件去下载；
            Response对象，停止process_request的执行，开始执行process_response
            Request对象，停止中间件的执行，将Request重新调度器
            raise IgnoreRequest异常，停止process_request的执行，开始执行process_exception
        """
        pass



    def process_response(self, request, response, spider):
        """
        spider处理完成，返回时调用
        :param response:
        :param result:
        :param spider:
        :return: 
            Response 对象：转交给其他中间件process_response
            Request 对象：停止中间件，request会被重新调度下载
            raise IgnoreRequest 异常：调用Request.errback
        """
        print('response1')
        return response

    def process_exception(self, request, exception, spider):
        """
        当下载处理器(download handler)或 process_request() (下载中间件)抛出异常
        :param response:
        :param exception:
        :param spider:
        :return: 
            None：继续交给后续中间件处理异常；
            Response对象：停止后续process_exception方法
            Request对象：停止中间件，request将会被重新调用下载
        """
        return None

import sys
from scrapy.cmdline import execute

if __name__ == '__main__':
    execute(["scrapy","crawl","chouti","--nolog"])

from scrapy.commands import ScrapyCommand
from scrapy.utils.project import get_project_settings


class Command(ScrapyCommand):

    requires_project = True

    def syntax(self):
        return '[options]'

    def short_desc(self):
        return 'Runs all of the spiders'

    def run(self, args, opts):
        spider_list = self.crawler_process.spiders.list()
        for name in spider_list:
            self.crawler_process.crawl(name, **opts.__dict__)
        self.crawler_process.start()

from scrapy import signals


class MyExtension(object):
    def __init__(self, value):
        self.value = value

    @classmethod
    def from_crawler(cls, crawler):
        val = crawler.settings.getint('MMMM')
        ext = cls(val)

        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)

        return ext

    def spider_opened(self, spider):
        print('open')

    def spider_closed(self, spider):
        print('close')

EXTENSIONS = {
   # 'scrapy.extensions.telnet.TelnetConsole': None,
   'xdb.ext.MyExtend': 666,
}

from scrapy.dupefilters import BaseDupeFilter

class DdbDuperFiler(BaseDupeFilter):
    """
    基于redis的去重: 1 完全自定义
    """

    def __init__(self):
        # 第一步: 连接redis
        self.conn = Redis(host='127.0.0.1', port='6379')

    def request_seen(self, request):
        """
        第二步: 检测当前请求是否已经被访问过
        :return: True表示已经访问过；False表示未访问过
        """
        # 方式一   直接将url添加进集合中作为判断
        # result = self.conn.sadd('urls', request.url)
        # if result == 1:
        #     return False
        # return True

        # 方式二   先fingerprint再将url添加进集合中作为判断
        fd = request_fingerprint(Request(url=request.url))
        result = self.conn.sadd('urls', fd)
        if result == 1:
            return False
        return True

DUPEFILTER_CLASS = 'ddb.dup.DdbDuperFiler'

# 连接方式1
# REDIS_HOST = '127.0.0.1'
# REDIS_PORT = '6379'
# REDIS_PARAMS  = {'password':'beta'}         # redis连接参数
# 连接方式2(优先级高于1)
REDIS_URL = 'redis://127.0.0.1:6379'   # 'redis://user:pass@hostname:9001'

REDIS_ENCODING = 'utf-8'
DUPEFILTER_CLASS = 'scrapy_redis.dupefilter.RFPDupeFilter'
DUPEFILTER_KEY = 'dupefilter:%(timestamp)s'     # 这是默认配置

from scrapy_redis.connection import get_redis_from_settings
from scrapy_redis.dupefilter import RFPDupeFilter
from scrapy_redis import defaults


class RedisDuperFiler(RFPDupeFilter):
    """
    基于redis_redis的去重: 3 实现自定义redis中存储的键的名字
    """

    @classmethod
    def from_settings(cls, settings):
        """Returns an instance from given settings.
        """
        server = get_redis_from_settings(settings)
        # XXX: This creates one-time key. needed to support to use this
        # class as standalone dupefilter with scrapy's default scheduler
        # if scrapy passes spider on open() method this wouldn't be needed
        # TODO: Use SCRAPY_JOB env as default and fallback to timestamp.
        key = defaults.DUPEFILTER_KEY % {'timestamp': 'xiaodongbei'}
        debug = settings.getbool('DUPEFILTER_DEBUG')
        return cls(server, key=key, debug=debug)

REDIS_HOST = '127.0.0.1'
REDIS_PORT = '6379'
# REDIS_PARAMS  = {'password':'beta'}         # redis连接参数
REDIS_ENCODING = 'utf-8'
DUPEFILTER_CLASS = 'ddb.dup.RedisDuperFiler'
DUPEFILTER_KEY = 'dupefilter:%(timestamp)s'     # 这是默认配置