python - 如何为每个项目调用 Parse_page2 方法

Question

我正在尝试为每个项目调用 parse_page2 方法。但是每次我运行这个蜘蛛时，我每页只能获取单个项目，所以我如何为每个项目调用 parse_page2 方法。

from scrapy.http import Request
from eScraper.items import EscraperItem
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.spiders import CrawlSpider

#------------------------------------------------------------------------------ 

class ESpider(CrawlSpider):
    name = "atisundarSpider"

    allowed_domains = ["atisundar.com"]

    URLSList = []

    for n in range (1,20):

        URLSList.append('http://atisundar.com/collections/sarees?page=' + str(n))
        URLSList.append('http://atisundar.com/collections/salwar-suits?page=' + str(n))

    start_urls = URLSList

    def parse(self, response):

        item = EscraperItem()
        hxs = HtmlXPathSelector(response)
        sites = hxs.select('//div[@class="block product size-medium"]')
        items = []

        for site in sites:
            item = EscraperItem()
            item['productDesc'] = "" 
            item['productSite'] = ["http://atisundar.com/"]
            item['productTitle'] = site.select('.//div[@class="main"]/a/@title').extract()
            item['productURL'] = ["http://atisundar.com" + site.select('.//div[@class="main"]/a/@href').extract()[0].encode('utf-8')]
            item['productPrice'] = site.select('.//p[@class="pricearea"]//span[@class="was-price"]/text()').extract() + site.select('.//p[@class="pricearea"]//span[@class="price"]/text()').extract() 
            item['productImage'] = [site.select('.//div[@class="main"]/a/img/@src').extract()[0].split('?')[0]] + [site.select('.//div[@class="main"]/a/img/@src').extract()[0].split('?')[0].replace("medium","grande")]
            item['image_urls'] = item['productImage']
            items.append(item)

            secondURL = "http://admin.atisundar.com/store/skuDetails?product_id=" + site.select('.//div[@class="main"]/a/text()').extract()[1].strip().split("#")[-1]

            request = Request(secondURL,
                      callback=self.parse_page2)
            request.meta['item'] = item 

            return request

    def parse_page2(self, response):

        item = response.meta['item']
        #item['other_url'] = response.url
        return item

Answer 1

1) 您没有使用 CrawlSpider 功能，我建议您从 BaseSpider 继承您的蜘蛛

2) 在for循环中

for site in sites:

使用 yield 而不是 return ，否则它将在第一次迭代中打破循环。

yield request

3) 在parse_page2中从response.request.meta获取item，而不是从response.meta

item = response.request.meta['item']

现在应该可以工作了。

from scrapy.http import Request
from eScraper.items import EscraperItem
from scrapy.selector import HtmlXPathSelector

#------------------------------------------------------------------------------ 
from scrapy.spider import BaseSpider


class ESpider(BaseSpider):
    name = "atisundarSpider"

    allowed_domains = ["atisundar.com"]

    URLSList = []

    for n in range (1,20):

        URLSList.append('http://atisundar.com/collections/sarees?page=' + str(n))
        URLSList.append('http://atisundar.com/collections/salwar-suits?page=' + str(n))

    start_urls = URLSList

def parse(self, response):
    item = EscraperItem()
    hxs = HtmlXPathSelector(response)
    sites = hxs.select('//div[@class="block product size-medium"]')
    for site in sites:
        item = EscraperItem()
        item['productDesc'] = "" 
        item['productSite'] = ["http://atisundar.com/"]
        item['productTitle'] = site.select('.//div[@class="main"]/a/@title').extract()
        item['productURL'] = ["http://atisundar.com" + site.select('.//div[@class="main"]/a/@href').extract()[0].encode('utf-8')]
        item['productPrice'] = site.select('.//p[@class="pricearea"]//span[@class="was-price"]/text()').extract() + site.select('.//p[@class="pricearea"]//span[@class="price"]/text()').extract() 
        item['productImage'] = [site.select('.//div[@class="main"]/a/img/@src').extract()[0].split('?')[0]] + [site.select('.//div[@class="main"]/a/img/@src').extract()[0].split('?')[0].replace("medium","grande")]
        item['image_urls'] = item['productImage']
        secondURL = "http://admin.atisundar.com/store/skuDetails?product_id=" + site.select('.//div[@class="main"]/a/text()').extract()[1].strip().split("#")[-1]
        request = Request(secondURL,
                  callback=self.parse_page2)
        request.meta['item'] = item 
        yield request

def parse_page2(self, response):

    item = response.request.meta['item']
    #item['other_url'] = response.url
    return item