python - 如何从网页中提取page_source

Question

我正在尝试从政府网页获取数据，但是，当我获取页面源时，它不包含浏览器中显示的数据。

from selenium import webdriver
from selenium.webdriver.support.ui import Select


page = 'http://web.cvm.gov.br/app/esforcosrestritos/#/consultarOferta'

driver = webdriver.Chrome()
driver.get(page)

## Click on "Encerrada"
driver.find_element_by_xpath('//*[@id="content"]/div[4]/div[2]/div/div /div[4]/div[2]/label[3]/input').click()


## Select year
year = Select(driver.find_element_by_xpath('//*[@id="content"]/div[4]/div[2]/div/div/div[4]/div[1]/div/select'))
year.select_by_visible_text('2017')


## Click on "Pesquisar"
driver.find_element_by_xpath('//*[@id="content"]/div[4]/div[3]/div/a[1]/span').click()


## Click on "DEBENTURES SIMPLES" inside "Ofertas Encerradas"
driver.find_element_by_css_selector('#content > div.container.ng-scope > div:nth-child(4) > div:nth-child(2) > div > table > tbody > tr:nth-child(15) > td.col-lg-2.text-left.ng-binding').click()

## Click on 1st result
driver.find_element_by_css_selector('#content > div.container.ng-scope > div:nth-child(4) > div > div > table > tbody > tr.text-center > td.text-left.ng-binding').click()

##Page Source
html = driver.page_source

在此示例中，第一个字段“CNPJ”，我得到的不是值“04.031.960/00001-70”，而是:

<input type="text" class="form-control ng-pristine ng-untouched ng-valid ng-valid-maxlength" data-ng-cnpj="" data-ng-model="$responsavel.ofertante.cnpj" data-ng-change="getNomeResponsavelPorCnpj($responsavel.ofertante)" data-ng-disabled="mesmosDadosEmissor || $responsavel.disabled" maxlength="18" disabled="disabled">

此外，如果我将鼠标悬停在浏览器中的值上，则无法选择它。

有没有办法从这种类型的页面获取数据？

Answer 1

我终于从浏览器日志中获取信息解决了这个问题。数据没有直接出现在 html 源中，而是在过程中使用的 POST 中。

这是最终的工作代码:

from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities 
import json
import pandas as pd


page = 'http://web.cvm.gov.br/app/esforcosrestritos/#/consultarOferta'

d = DesiredCapabilities.CHROME
d['loggingPrefs'] = { 'performance':'ALL' }
driver = webdriver.Chrome(desired_capabilities=d)
driver.get(page)

## Click on "Encerrada"
driver.find_element_by_xpath('//*[@id="content"]/div[4]/div[2]/div/div /div[4]/div[2]/label[3]/input').click()


## Select year
year = Select(driver.find_element_by_xpath('//*[@id="content"]/div[4]/div[2]/div/div/div[4]/div[1]/div/select'))
year.select_by_visible_text('2017')


## Click on "Pesquisar"
driver.find_element_by_xpath('//*[@id="content"]/div[4]/div[3]/div/a[1]/span').click()


## Click on "DEBENTURES SIMPLES" inside "Ofertas Encerradas"
driver.find_element_by_css_selector('#content > div.container.ng-scope > div:nth-child(4) > div:nth-child(2) > div > table > tbody > tr:nth-child(15) > td.col-lg-2.text-left.ng-binding').click()

## Click on 1st result
driver.find_element_by_css_selector('#content > div.container.ng-scope > div:nth-child(4) > div > div > table > tbody > tr.text-center > td.text-left.ng-binding').click()


## Selenium browser log
performance_log = driver.get_log('performance')

## Find log with allocation information
for j in range(len(performance_log)):
    if performance_log[j]['message'].find('Clubes de Investimento') != -1:
        break

allocation = performance_log[j]['message']

## Filter allocation data
allocation = allocation.replace('\\', '')
allocation = allocation[allocation.find('{"colocacoes":['):]


## Put data into a Pandas DataFrame
allocation_table = pd.DataFrame(columns = ['tipoInvestidor', 'numeroInvestidores', 'quantidadeValorMobiliario'])
slice_allocation = '{"tipoInvestidor":{"id":' 
slice_alternative= '{"numeroInvestidores":' 

for i in range(1,11):

    beginning = allocation.find(slice_allocation+str(i)) if allocation.find(slice_allocation+str(i))!=-1 else allocation.find(slice_alternative) 
    end = allocation.find(slice_allocation+str(i+1)) if allocation.find(slice_allocation+str(i+1))!=-1 else allocation.find(slice_alternative) 

    allocation_investor = allocation[beginning:end-1]
    allocation = allocation[end:]

    allocation_investor = json.loads(allocation_investor)
    allocation_investor['tipoInvestidor'] = allocation_investor['tipoInvestidor']['descricao']

    allocation_table = allocation_table.append(allocation_investor, ignore_index = True)

allocation_table.fillna(0, inplace = True)