python - Scrapy - 发送 AJAX FormRequest 返回错误 419

Question

https://unicodono.com.br/anuncios?page=3&uf_id=&cidade_id=&marca=&modelo=&versao=&cambio=&valor_minimo=&valor_maximo=&ano_minimo=&ano_maximo=&km_minimo=&km_maximo=&usado=&novos=&moto=&carro=&orderby=menor_valor&blindagem=&nao_blindagem=&cep=

这是我开始抓取过程的 URL。

https://unicodono.com.br/anuncios/itens/ajax?page=1

这是将数据动态加载到页面的 AJAX 调用。

我认为我正确发送了表单和 header ，但响应返回错误 419 并且爬虫停止运行。我不知道如何解释这个错误。我的 spider 是不是遗漏了什么？

class MySpider(CrawlSpider):

    name = 'myspider'    

    start_urls = ['https://unicodono.com.br/anuncios?page=1&uf_id=&cidade_id=&marca=&modelo=&versao=&cambio=&valor_minimo=&valor_maximo=&ano_minimo=&ano_maximo=&km_minimo=&km_maximo=&usado=&novos=&moto=&carro=&orderby=menor_valor&blindagem=&nao_blindagem=&cep=',]

    form_data = {'uf_id': '',
                 'cidade_id': '',
                 'marca': '',
                 'modelo': '',
                 'versao': '',
                 'valor_minimo': '',
                 'valor_maximo': '',
                 'ano_minimo': '',
                 'ano_maximo': '',
                 'km_minimo': '',
                 'km_maximo': '',
                 'orderby': 'menor_valor'}

    def parse(self, response):
        for url in self.start_urls:
            yield scrapy.FormRequest(
                url='https://unicodono.com.br/anuncios/itens/ajax?page=1',
                method='POST',
                headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
                         'Referer': url},
                callback=self.parse_page,
                formdata=self.form_data
            )

    def parse_page(self, response):

        logging.info('parse_page function called on %s', response.url)

        from scrapy.shell import inspect_response
        inspect_response(response, self)

        yield {'data': response.text}

Answer 1

使用请求库

import requests

form_data = {'orderby': 'menor_valor'}

headers = {
    "Cookie": "ga=GA1.3.500010858.1561116466; _gid=GA1.3.1602312084.1561116466; _gat_gtag_UA_15308183_2=1; XSRF-TOKEN=eyJpdiI6IlZQaFZua1pFUW9ORDFlNTV0UGpnT3c9PSIsInZhbHVlIjoiMkpPd1lac1VTNzcrV2hETzk0V3grcFNrSnA2eEJ3SmdUZkpIUGdIQjNCa01tdWJNdDI4VlR4ODlkVlVTemRcLzUiLCJtYWMiOiJhY2I1NDJiYmFmZDA2MWNlNTQ5NGJmYjZhNDM3NTExMTIzZDYyYTY5YjM3MmJhZWE1NTE1MzA0MGNmMjY5M2M1In0%3D; unicodono_session=eyJpdiI6ImhyWE8xbGhtQVBacnpyTTJ6NmxPanc9PSIsInZhbHVlIjoieEwybHBaNTFzaWR0elplRjcxWHc2RFJjK1Q1WlJmZmFsdGVyZVZtaEhPcmdrNVQ1bVZpZFBoS2RuNDVreEhBWCIsIm1hYyI6ImIyZWNhNWE5ODE1YjU5OTEyNjRkNWQ4ZTg5ZmMwOTVmNWEyYjhiMzE0MzJmODE4OWM3NTQ2ZTNmOTliMzZhNjQifQ%3D%3D",
    "X-CSRF-TOKEN": "R8UOoGWhksZEUJsdmIsQwpA9Gx9qSTXpvIgBZcXX"
}

url='https://unicodono.com.br/anuncios/itens/ajax?page=1'

response = requests.post(url,data=form_data,headers=headers)
print(response.json())