python - 从不同页面获取元素时如何避免 StaleElementReferenceError？

Question

我想获得比赛的所有结果。该网站显示 50 行/页。我使用 selenium 导航到下一页(带有后缀 #page-x 的相同 URL)，但每当我尝试在下一页上查找元素(表格单元格 = td)时，都会收到 StaleElementReferenceException 错误。

我尝试在步骤之间关闭驱动程序，以便一次只获取一个元素列表。我还尝试使用 URL+后缀单独加载页面，但无法正确加载。我尝试过构建单独的列表(一开始我想要一个包含所有结果的大列表)。

from selenium import webdriver
url = "https://tickets.justrun.ca/quidchrono.php?a=qcResult&raceid=8444"

#The block under works well and I get a list of cells as intended.
driver = webdriver.Chrome()
driver.maximize_window()
driver.get(url)
elements = driver.find_elements_by_tag_name("td")
course = []
for i in range(len(elements)):
    course.append(elements[i].text)

to_2 = driver.find_element_by_link_text("2")
to_2.click()
print(driver.current_url)

#I'm trying similar code for the next chunk, but it doesn't work.
elements2 = driver.find_elements_by_tag_name("td")
print(len(elements2))
print(elements2[5].text)
course2 = []
for i in range(len(elements2)):
    course2.append(elements2[i].text)
driver.close()

我希望得到一个新列表(course2)，其中包含第二页的结果，但我收到了陈旧元素错误。当我打印当前 URL 时，结果符合预期。当我打印 len(elements2) 时，也可以。看起来问题出在我尝试获取元素的文本时。

Answer 1

解决方案 1:

使用BeautifulSoup和selenium，WebDriverWait正在等待某个条件发生，然后再继续执行代码。有关 BeautifulSoup 的更多详细信息.

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup

url = "https://tickets.justrun.ca/quidchrono.php?a=qcResult&raceid=8444"
driver = webdriver.Chrome()

driver.get(url)

data = []
while True:
    course = []
    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "tableJustrun")))

    page_soup = BeautifulSoup(driver.page_source, 'lxml')
    # get table data 
    tbody = page_soup.find("tbody",{"id":"searchResultBoxParticipants"})
    rows = tbody.find_all("tr")

    for row in rows:
        rowData = []
        for td in row.find_all("td"):
            rowData.append(td.text)

        course.append(rowData)
    data.append(course)

    try:
        pagination = driver.find_element_by_class_name("simple-pagination")
        next_page = pagination.find_element_by_link_text("Suivant")
        # iterate next page
        next_page.click()
    except Exception as e:
        break

print(data)

解决方案 2:

使用pandas库。

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd

url = "https://tickets.justrun.ca/quidchrono.php?a=qcResult&raceid=8444"
driver = webdriver.Chrome()
driver.get(url)

data = []
while True:
    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "tableJustrun")))
    tables = pd.read_html(driver.page_source)
    #append Participants table data
    data.append(tables[0])

    try:
        pagination = driver.find_element_by_class_name("simple-pagination")
        next_page = pagination.find_element_by_link_text("Suivant")
        # iterate next page
        next_page.click()
    except Exception as e:
        break

#Concat dataframe object
result = pd.concat(data)
print(result)