python - 使用 Selenium 单击 'Load More' 按钮直到它不存在(Youtube)

Question

我正在尝试编写一个转到 YouTube channel 的 Python 脚本。单击“视频”选项卡。然后抓取网页内容。我完全可以抓取内容，直到...我找到加载更多按钮。我设法单击加载更多按钮一次的 python 脚本，但它再也不会按下它:'( 我如何修改代码，我必须让它一次又一次地单击它，直到它不存在。这样，我可以打通用户的完整 channel ，从用户的每一个视频中获取信息。谢谢。

from selenium import webdriver      
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

^^ 这些是我导入的模块。我也不知道如何将 NoSuchElementException 添加到我的代码中。这是代码:

chrome_path = r"/Users/jack/Desktop/Other/Downloads/Software_and_Programs/chromedriver"
browser = webdriver.Chrome(chrome_path)
YOUTUBER_HOME_PAGE_URL = "https://www.youtube.com/user/Google/videos"
PATIENCE_TIME = 60
LOAD_MORE_BUTTON_XPATH = '//*[@id="browse-itemsprimary"]/li[2]/button/span/span[2]' 

def waitForLoad(inputXPath):
    Wait = WebDriverWait(browser, PATIENCE_TIME)
    Wait.until(EC.presence_of_element_located((By.XPATH, inputXPath)))

loadMoreButtonExists = True
while loadMoreButtonExists:
    try:
        waitForLoad(LOAD_MORE_BUTTON_XPATH)
        WebDriverWait(browser, PATIENCE_TIME)
        loadMoreButton = browser.find_element_by_partial_link_text('Load More')
        #loadMoreButton = browser.find_element_by_xpath(LOAD_MORE_BUTTON_XPATH)
        loadMoreButton.click()
    except:
        print 'we have completely loaded every video from this Youtuber. Now we will scrape the video content\n'
        loadMoreButtonExists = False

我已经使用了 xpath 方式，但似乎仍然无法正常工作。我已经在上面的代码中注释掉了。如果我能在这方面得到帮助，那就太棒了。我找不到任何好的答案。我相信它可以用 Selenium 来完成，但如果不能，我应该使用什么？

Answer 1

你可以使用下面的代码来滚动

from selenium import webdriver      
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time


#browser = webdriver.Firefox()#Chrome('./chromedriver.exe')
YOUTUBER_HOME_PAGE_URL = "https://www.youtube.com/user/Google/videos"
PATIENCE_TIME = 60
LOAD_MORE_BUTTON_XPATH = '//*[@id="browse-itemsprimary"]/li[2]/button/span/span[2]' 

driver = webdriver.Chrome('./chromedriver.exe')
driver.get(YOUTUBER_HOME_PAGE_URL)

while True:
    try:
        loadMoreButton = driver.find_element_by_xpath("//button[contains(@aria-label,'Load more')]")
        time.sleep(2)
        loadMoreButton.click()
        time.sleep(5)
    except Exception as e:
        print e
        break
print "Complete"
time.sleep(10)
driver.quit()