java - 使用 Selenium 迭代网站的所有链接

Question

我是 Selenium 新手，我想下载所有 pdf、ppt(x) 和 doc(x) 文件来自网站。我编写了以下代码。但我很困惑如何获取内部链接:

import java.io.*;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.openqa.selenium.By;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.firefox.FirefoxDriver;

public class WebScraper {

    String loginPage = "https://blablah/login";
    static String userName = "11";
    static String password = "11";
   static String mainPage = "https://blahblah";

    public WebDriver driver = new FirefoxDriver();
    ArrayList<String> visitedLinks = new ArrayList<>();

    public static void main(String[] args) throws IOException {

        System.setProperty("webdriver.gecko.driver", "E:\\geckodriver.exe");

        WebScraper webSrcaper = new WebScraper();
        webSrcaper.openTestSite();
        webSrcaper.login(userName, password);

        webSrcaper.getText(mainPage);
        webSrcaper.saveScreenshot();
        webSrcaper.closeBrowser();
    }

        /**
     * Open the test website.
     */
    public void openTestSite() {

        driver.navigate().to(loginPage);
    }

    /**
     * @param username
     * @param Password Logins into the website, by entering provided username and password
     */
    public void login(String username, String Password) {

        WebElement userName_editbox = driver.findElement(By.id("IDToken1"));
        WebElement password_editbox = driver.findElement(By.id("IDToken2"));
        WebElement submit_button = driver.findElement(By.name("Login.Submit"));

        userName_editbox.sendKeys(username);
        password_editbox.sendKeys(Password);
        submit_button.click();

    }

    /**
     * grabs the status text and saves that into status.txt file
     *
     * @throws IOException
     */
    public void getText(String website) throws IOException {

        driver.navigate().to(website);

        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        List<WebElement> allLinks = driver.findElements(By.tagName("a"));

        System.out.println("Total no of links Available: " + allLinks.size());

        for (int i = 0; i < allLinks.size(); i++) {

            String fileAddress = allLinks.get(i).getAttribute("href");

            System.out.println(allLinks.get(i).getAttribute("href"));
            if (fileAddress.contains("download")) {
                driver.get(fileAddress);
            } else {
//                getText(allLinks.get(i).getAttribute("href"));
            }
        }
        
    }

    /**
     * Saves the screenshot
     *
     * @throws IOException
     */
    public void saveScreenshot() throws IOException {
        File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
        FileUtils.copyFile(scrFile, new File("screenshot.png"));
    }

    public void closeBrowser() {
        driver.close();
    }
    
}

我有一个 if 子句，用于检查当前链接是否是可下载文件(地址包含“下载”一词)。如果是的话，我会得到它，如果不是，该怎么办？那部分是我的问题。我尝试实现递归函数来检索嵌套链接并重复嵌套链接的步骤，但没有成功。

同时，当输入 https://blahblah 作为输入时找到的第一个链接是 https://blahblah/#，它指的是与 https://blahblah 相同的页面。它也可能会导致问题，但目前，我陷入了另一个问题，即递归函数的实现。你能帮我一下吗？

Answer 1

您离得并不远，但是要回答您的问题，请将所有链接抓取到元素列表中，迭代并单击(然后等待)。使用 C# 是这样的；

       IList<IWebElement> listOfLinks = _driver.FindElements(By.XPath("//a"));
        foreach (var link in listOfLinks)
        {
            if(link.GetAttribute("href").Contains("download"))
            {
            link.Click();
            WaitForSecs(); //Thread.Sleep(1000)
            }
        }

JAVA

    List<WebElement> listOfLinks = webDriver.findElements(By.xpath("//a"));
    for (WebElement link :listOfLinks ) {

        if(link.getAttribute("href").contains("download"))
        {
            link.click();
            //WaitForSecs(); //Thread.Sleep(1000)
        }
    }