python - 如何更改 ItemLoader 的选择器

Question

我正在尝试使用 ItemLoader 解析来自多个页面的数据来填充项目。但正如我现在所看到的，我无法更改初始化 ItemLoader 时使用的选择器。和documentation关于选择器属性的说明:

selector

The Selector object to extract data from. It’s either the selector given in the constructor or one created from the response given in the constructor using the default_selector_class. This attribute is meant to be read-only.

这里是示例代码:

def parse(self, response):
    sel = Selector(response)
    videos = sel.xpath('//div[@class="video"]')

    for video in videos:
        loader = ItemLoader(VideoItem(), videos)
        loader.add_xpath('original_title', './/u/text()')
        loader.add_xpath('original_id', './/a[@class="hRotator"]/@href', re=r'movies/(\d+)/.+\.html')

        try:
            url = video.xpath('.//a[@class="hRotator"]/@href').extract()[0]
            request = Request(url,
                      callback=self.parse_video_page)
        except IndexError:
            pass

        request.meta['loader'] = loader
        yield request

    pages = sel.xpath('//div[@class="pager"]//a/@href').extract()
    for page in pages:
        url = urlparse.urljoin('http://www.mysite.com/', page)
        request = Request(url, callback=self.parse)
        yield request

def parse_video_page(self, response):
    loader = response.meta['loader']
    sel = Selector(response)

    loader.add_xpath('original_description', '//*[@id="videoInfo"]//td[@class="desc"]/h2/text()')
    loader.add_xpath('duration', '//*[@id="video-info"]/div[2]/text()')
    loader.add_xpath('tags', '//*[@id="tags"]//a/text()')

    item = loader.load_item()

    return item

目前，我无法从第二页抓取信息。

Answer 1

直接回答您的问题 - 要更改 ItemLoader 的选择器，您可以将新的选择器对象设置为 loader.selector 属性。

def parse_video_page(self, response):
    loader = response.meta['loader']
    sel = Selector(response)
    loader.selector = sel

    loader.add_xpath(
        'original_description', 
        '//*[@id="videoInfo"]//td[@class="desc"]/h2/text()'
    )
    # ...

但是这种使用加载器对象的方式似乎是意外的，因此 - 不支持 - 库更新可能会破坏此代码或产生意外的错误。另外，将加载程序传递给请求元也是一件坏事，因为加载程序对象引用响应对象 - 这在某些情况下可能会导致内存问题。

在多个回调中收集项目字段的更正确方法如下(请注意注释):

def parse(self, response):
    sel = Selector(response)
    videos = sel.xpath('//div[@class="video"]')

    for video in videos:
        try:
            url = video.xpath('.//a[@class="hRotator"]/@href').extract()[0]
        except IndexError:
            continue
        loader = ItemLoader(VideoItem(), videos)
        loader.add_xpath('original_title', './/u/text()')
        loader.add_xpath(
            'original_id', 
            './/a[@class="hRotator"]/@href', 
            re=r'movies/(\d+)/.+\.html'
        )
        item = loader.load_item()
        yield Request(
            urlparse.urljoin(response.url, url),
            callback=self.parse_video_page,
            # Note: item passed to the meta dict, not loader itself
            meta={'item': item}
        )

    pages = sel.xpath('//div[@class="pager"]//a/@href').extract()
    for page in pages:
        url = urlparse.urljoin('http://www.mysite.com/', page)
        yield Request(url, callback=self.parse)

def parse_video_page(self, response):
    item = response.meta['item']

    # Note: new loader object created, 
    # item from response.meta is passed to the constructor
    loader = ItemLoader(item, response=response)
    loader.add_xpath(
        'original_description', 
        '//*[@id="videoInfo"]//td[@class="desc"]/h2/text()'
    )
    loader.add_xpath(
        'duration', 
        '//*[@id="video-info"]/div[2]/text()'
    )
    loader.add_xpath('tags', '//*[@id="tags"]//a/text()')
    return loader.load_item()