python - 抓取隐藏框架 JavaScript

Question

我正在尝试抓取隐藏框架内的数据；框架如下所示

<!-- Content of the details tabs here  -->
    <div id="tabDetail_0" class="tab_content tab_detail" style="display: 
    block;"><iframe id="iframe_0" src="https://www.tmdn.org/tmview/get- 
    detail?st13=GB500000003342197" width="100%" height="600px;" 
    frameborder="0"></iframe></div></div></div> <!-- resultTabs -->

正如您所看到的 HTML 中有一个链接，我尝试打开一个新的 webdriver 实例并导航该链接并获取数据，它有效，但网站停止了，因为直接导航这些链接是不允许或限制的。

import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
import traceback
import time

option = webdriver.ChromeOptions()
chrome_prefs = {}
option.experimental_options["prefs"] = chrome_prefs
chrome_prefs["profile.default_content_settings"] = {"images": 2}
chrome_prefs["profile.managed_default_content_settings"] = {"images": 2}



url ="https://www.tmdn.org/tmview/welcome#"
xlsName = 'D:\\test.xlsx' 
records = []

start_time = time.time()

driver = webdriver.Chrome(executable_path="D:\Python\chromedriver.exe",chrome_options=option)
driver.get(url)
time.sleep(10)
driver.find_element_by_xpath('//*[@id="buttonBox"]/a').click()
time.sleep(10)

x=-1

try:

    #click advanced search
    driver.find_element_by_name("lnkAdvancedSearch").click()
    #
    time.sleep(5)
    #to select Designated territories
    driver.find_element_by_id('DesignatedTerritories').click()
    time.sleep(5)   
    TerritoryLabelElements = driver.find_elements_by_css_selector( 'div.optEUGroupContainer label')
    for elem in TerritoryLabelElements:
        if elem.text == 'United Kingdom':
            elem.click()
    time.sleep(5)
    driver.find_element_by_id('DesignatedTerritories').click()
    #
    time.sleep(5)
    #to select from Trade mark offices
    driver.find_element_by_id('SelectedOffices').click()
    time.sleep(5)   
    TerritoryLabelElements = driver.find_elements_by_css_selector( 'div.multiSelectOptions label')
    for elem in TerritoryLabelElements:
        if elem.text == 'GB United Kingdom ( UKIPO )':
            elem.click()
    time.sleep(5)
    driver.find_element_by_id('SelectedOffices').click()
    #Trade mark status
    driver.find_element_by_id('TradeMarkStatus').click()
    time.sleep(5)   
    TerritoryLabelElements = driver.find_elements_by_css_selector( 'div.multiSelectOptions label')
    for elem in TerritoryLabelElements:
        if elem.text == 'Filed':
            elem.click()
        if elem.text == 'Registered':
            elem.click()
    time.sleep(5)
    driver.find_element_by_id('TradeMarkStatus').click()
    # dates
    startdate = driver.find_element_by_id("ApplicationDateFrom")
    startdate.clear()
    startdate.send_keys ('01-10-2018')
    enddate = driver.find_element_by_id("ApplicationDateTo")
    enddate.clear()
    enddate.send_keys ('31-10-2018' )   
    # click search
    time.sleep(5)
    driver.find_element_by_id("SearchCopy").click()
    time.sleep(5)
    html= driver.page_source
    soup = BeautifulSoup(html,'html.parser')
    tbl = soup.find("table", id="grid") 


    driver.find_element_by_link_text('100').click()
    time.sleep(5)
    # #LOOP
    for i in range(1, 73):
        html= driver.page_source
        soup = BeautifulSoup(html,'html.parser')
        tbl = soup.find("table", id="grid")
        #extract data from table using soup
        tr_rows = tbl.find_all('tr')
        for tr_row in tr_rows[1:]:

            td_cells=tr_row.find_all('td')

            Trade_mark_name=td_cells[4].text
            Trade_mark_office=td_cells[5].text
            Designated_territory=td_cells[6].text
            Application_number=td_cells[7].text
            Registration_number=td_cells[8].text
            Trade_mark_status=td_cells[9].text
            Trade_mark_type=td_cells[13].text
            Applicant_name=td_cells[11].text
            Nice_class=td_cells[10].text
            Application_date=td_cells[12].text
            Registration_date=td_cells[14].text

            x=x+1
            #Click indiviual links
            el=driver.find_elements_by_class_name('cell_tmName_column')[x]

            action = webdriver.common.action_chains.ActionChains(driver)
            action.move_to_element_with_offset(el, 0, 0)
            action.click()
            action.perform()
            time.sleep(3)
            #switch to iframe of tab details
            iframe = driver.find_elements_by_tag_name('iframe')[0]

            driver.switch_to.frame(iframe)
            #get data from iframe 
            html2= driver.page_source                       
            soup2 = BeautifulSoup(html2,'html.parser')
            tblOwner = soup2.find("div", id="anchorOwner").find_next('table')
            tblRep = soup2.find("div", id="anchorRepresentative").find_next('table')
            # then switch back:
            driver.switch_to.default_content()

            try:
                Owner_Address= tblOwner.find("td", text="Address").find_next('td')
            except:
                Owner_Address='No Entry'

            try:
                Representative_Name=tblRep.find("td", text="Name").find_next('td').text.strip()
            except:
                Representative_Name='No Entry'

            records.append((Designated_territory,Applicant_name,Trade_mark_name,Application_date,Application_number,Trade_mark_type, Nice_class,Owner_Address,Trade_mark_office,    Registration_number,Trade_mark_status,Registration_date,Representative_Name))
            time.sleep(1)


            driver.find_elements_by_css_selector( 'a.close_tab')[0].click()

        #navigate next page_source
        driver.find_element_by_id('next_t_grid_toppager').click()
        time.sleep(2)
        x=-1
    #LOOP

    df = pd.DataFrame(records, columns=['Designated_territory','Applicant_name','Trade_mark_name','Application_date','Application_number','Trade_mark_type', 'Nice_class','Owner_Address','Trade_mark_office',  'Registration_number','Trade_mark_status','Registration_date','Representative_Name'])  
    df.to_excel(xlsName,sheet_name='sheet1', index=False, encoding='utf-8') 


except Exception:

    df = pd.DataFrame(records, columns=['Designated_territory','Applicant_name','Trade_mark_name','Application_date','Application_number','Trade_mark_type', 'Nice_class','Owner_Address','Trade_mark_office',  'Registration_number','Trade_mark_status','Registration_date','Representative_Name'])  
    df.to_excel(xlsName,sheet_name='sheet1', index=False, encoding='utf-8')

    traceback.print_exc()

time.sleep(5)
driver.quit()

Answer 1

您需要做的是switch_to.frame:

iframe = driver.find_element_by_xpath('//iframe[@id="iframe_0"]')
driver.switch_to.frame(iframe)
# than switch back:
driver.switch_to.default_content()

编辑:

您已经询问了 id 是否会更改该怎么办，因此您可以在 xpath 中使用 contains ，如下所示:

# this will find any iframe with and id of iframe_ 
# you should check there is only one, you can do so with: `iframes = driver.find_elements_by_xpath('//iframe[contains(@id,"iframe_")]')`
# than `print(len(iframes))` to see the amount of iframes 
iframe = driver.find_element_by_xpath('//iframe[contains(@id,"iframe_")]')
driver.switch_to.frame(iframe)
# than switch back:
driver.switch_to.default_content()

在您的代码中使用:

import time
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select

url ="https://www.tmdn.org/tmview/welcome#"

driver = webdriver.Chrome(executable_path=r"D:\New Proj\chromedriver.exe")
driver.get(url)
time.sleep(3)
driver.find_element_by_xpath('//*[@id="buttonBox"]/a').click()
time.sleep(3)

#Click advanced search
driver.find_element_by_name("lnkAdvancedSearch").click()
#
time.sleep(5)
#to select Designated territories
driver.find_element_by_id('DesignatedTerritories').click()
time.sleep(5)   
TerritoryLabelElements = driver.find_elements_by_css_selector( 'div.optEUGroupContainer label')
for elem in TerritoryLabelElements:
    if elem.text == 'United Kingdom':
        elem.click()
time.sleep(5)
driver.find_element_by_id('DesignatedTerritories').click()
#
time.sleep(5)
#to select from Trade mark offices
driver.find_element_by_id('SelectedOffices').click()
time.sleep(5)   
TerritoryLabelElements = driver.find_elements_by_css_selector( 'div.multiSelectOptions label')
for elem in TerritoryLabelElements:
    if elem.text == 'GB United Kingdom ( UKIPO )':
        elem.click()
time.sleep(5)
driver.find_element_by_id('SelectedOffices').click()
#Trade mark status
driver.find_element_by_id('TradeMarkStatus').click()
time.sleep(5)   
TerritoryLabelElements = driver.find_elements_by_css_selector( 'div.multiSelectOptions label')
for elem in TerritoryLabelElements:
    if elem.text == 'Filed':
        elem.click()
    if elem.text == 'Registered':
        elem.click()
time.sleep(5)
driver.find_element_by_id('TradeMarkStatus').click()
# dates
startdate = driver.find_element_by_id("ApplicationDateFrom")
startdate.clear()
startdate.send_keys ('10-01-2018')
enddate = driver.find_element_by_id("ApplicationDateTo")
enddate.clear()
enddate.send_keys ('10-01-2018' )   
# click search
time.sleep(5)
driver.find_element_by_id("SearchCopy").click()
time.sleep(30)

#Click first link
el=driver.find_elements_by_class_name('cell_tmName_column')[0]
action = ActionChains(driver)
action.move_to_element_with_offset(el, 0, 0)
action.click()
action.perform()
time.sleep(10)
iframe = driver.find_element_by_xpath('//iframe[@id="iframe_0"]')
driver.switch_to.frame(iframe)
# do something here I am printing the HTML
print(iframe.get_attribute('innerHTML'))
# than switch back:
driver.switch_to.default_content()

希望这对您有帮助!