python - 如何使用 selenium 网络驱动程序抓取网站而不会被阻止

Question

我正在抓取此页面 https://www.elcorteingles.es/supermercado/alimentacion-general/但每次浏览器都无法加载页面或无法访问网站。我该如何解决这个问题？

class SuperSpider(scrapy.Spider):
name = 'super'
allowed_domains = ['www.elcorteingles.es/supermercado']
start_urls = ['https://www.elcorteingles.es/supermercado/alimentacion-general/']

def __init__(self):
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    chrome_path = which("chromedriver")
    driver = webdriver.Chrome(executable_path=chrome_path)
    driver.get("https://www.elcorteingles.es/supermercado/alimentacion-general/")
    driver.maximize_window()
    time.sleep(25)
    self.html = driver.page_source
    driver.close()

def parse(self, response):
    pass

Answer 1

from fake_useragent import UserAgent
ua = UserAgent()
a = ua.random
user_agent = ua.random
print(user_agent)
options.add_argument(f'user-agent={user_agent}')

options.add_argument('--disable-blink-features=AutomationControlled')

options.add_argument('--headless')
options.add_argument("--window-size=1920,1080")
#your code
time.sleep(30)
print(driver.page_source)

这应该绕过 bot 检测，但要注意 driver_page_source 很大。