python-3.x - python+selenium+phantomjs 无法处理 `adding certification is strongly advised` 的 https url

Question

我尝试使用 python+selenium+phantomjs 处理 https 请求，但无法从服务器收到响应，这表明我强烈建议添加认证，这是我的 get_request 函数。

def get_random_x_forwarded_for():
    # 得到随机x-forwarded-for值
    numbers = []
    while not numbers or numbers[0] in (10, 172, 192):
        numbers = random.sample(range(1, 255), 4)
    return '.'.join(str(_) for _ in numbers)

def get_random_ua():
    # 得到随机user-agent值
    import os
    if os.path.exists("dicts/user-agent.txt"):
        f = open(ModulePath + "dicts/user-agents.txt", "r+")
        all_user_agents = f.readlines()
        f.close()
    else:
        all_user_agents = [
            "Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1)",
            "Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1; .NET CLR 3.0.04506.30)",
            "Mozilla/4.0 (Windows; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)",
            "Mozilla/4.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.33 Safari/532.0",
            "Mozilla/4.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.59 Safari/525.19",
            "Mozilla/4.0 (compatible; MSIE 6.0; Linux i686 ; en) Opera 9.70",
        ]
    random_ua_index = random.randint(0, len(all_user_agents) - 1)
    ua = re.sub(r"(\s)$", "", all_user_agents[random_ua_index])
    return ua

def get_request(url, by="MechanicalSoup", proxyUrl="", cookie=""):
    code = None
    title = None
    content = None
    if by == "seleniumPhantomJS":
        hasFormAction=False
        formActionValue=""
        from selenium import webdriver
        from selenium.common.exceptions import TimeoutException
        import time
        if proxyUrl == "" or proxyUrl == 0:
            service_args_value = ['--ignore-ssl-errors=true', '--ssl-protocol=any']
        if proxyUrl != "" and proxyUrl != 0:
            proxyType = proxyUrl.split(":")[0]
            proxyValueWithType = proxyUrl.split("/")[-1]
            service_args_value = ['--ignore-ssl-errors=true', '--ssl-protocol=any',
                                  '--proxy=%s' % proxyValueWithType, '--proxy-type=%s' % proxyType]
        # final_url=driver.current_url
        # print("正在访问的url是这个:\n"+final_url)
        # driver.quit()

        try:
            from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
            if cookie != "":
                dcap = dict(DesiredCapabilities.PHANTOMJS)
                dcap["phantomjs.page.settings.cookie"] = cookie
                dcap["phantomjs.page.settings.userAgent"] = get_random_ua()
                driver = webdriver.PhantomJS(service_args=service_args_value, desired_capabilities=dcap)
            else:
                driver = webdriver.PhantomJS(service_args=service_args_value)

            print(111111111)
            driver.implicitly_wait(5)
            driver.set_page_load_timeout(5)

            driver.get(url)
            print(222222222222222)
            # http://www.cnblogs.com/fnng/p/3269450.html
            originalCookie = driver.get_cookies()

            #print("current cookie is:\n" + str(originalCookie))

            import random
            code = 200 
            title = driver.title
            content = driver.page_source
            a=re.search(r'''(<.*type=('|")?submit('|")?.*>)''',content,re.I)
            if a:
                hasFormAction=True
                print(a.group(1))
                input(67666)
            else:
                pass

            print("len content is :\n" + str(len(content)))
            print("title is :\n" + title)

            if re.search(r"(页面不存在)|(未找到页面)|(page not found)|(404)",title+content,re.I):
                return get_request(url,by="MechanicalSoup")
            # time.sleep(5) # Let the user actually see something!
            # driver.quit()

        except TimeoutException as e:
            # Handle your exception here
            print(e)
        finally:
            driver.quit()

        return {
            'code': code,
            'title': title,
            'content': content,
            #True or False
            'hasFormAction':hasFormAction,
            #eg,https://www.baidu.com^a=1&b=2
            #eg,https://www.baidu.com/?a=1&b=2
            'formActionValue':formActionValue}

    else:
        import mechanicalsoup

        try:
            browser = mechanicalsoup.Browser(soup_config={"features": "lxml"})
            ua = get_random_ua()
            browser.session.headers.update({'User-Agent': '%s' % ua})
            # headers=browser.session.headers
            # if 'Cookie' in headers:
            # originalCookie=headers['Cookie']
            if cookie == "":
                pass
            else:
                browser.session.headers.update({'Cookie': '%s' % cookie})
            # print(originalCookie)
            x_forwarded_for = get_random_x_forwarded_for()
            browser.session.headers.update(
                {'X-Forwarded-For': '%s' % x_forwarded_for})


            result = browser.get(url,
                    timeout=10,verify=False)
            # print(dir(result))
            code = result.status_code
            content = result.content
            import chardet
            bytesEncoding = chardet.detect(content)['encoding']
            # print(bytesEncoding)
            content = content.decode(bytesEncoding)
            title = BeautifulSoup(content, "lxml").title
            if title is not None:
                title_value = title.string
            else:
                title_value = None
        except:

            code = 0
            title_value = "you may be blocked or the code doesn't handle ssl certificate well"
            content = 'can not get html content this time,may be blocked by the server to request'

        return_value = {
            'code': code,
            'title': title_value,
            'content': content}
        # print("访问当前url为:\n\t"+url+"\ntitle如下:")
        # print("\t"+str(return_value['title']))
        return return_value

b=get_request("https://www.zoomeye.org/search/advanced",by="seleniumPhantomJS")
#b=get_request("https://www.zoomeye.org/search/advanced",by="MechanicalSoup")
print(b)

当我使用b=get_request("https://www.zoomeye.org/search/advanced",by="seleniumPhantomJS")，它只会打印 11111..1 并卡在那里没有打印 2222.2，然后我尝试使用 b=get_request("https://www.zoomeye.org/search/advanced",by="MechanicalSoup")，这次我得到以下错误调试信息:


/usr/local/lib/python3.6/site-packages/requests/packages/urllib3/connectionpool.py:852: InsecureRequestWarning: 正在发出未经验证的 HTTPS 请求。强烈建议添加证书验证。请参阅:https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
不安全请求警告)


我想用python3+selenium+phantomjs来处理这个adding certificate verification is strongly advised错误，有人能帮帮我吗？

后来尝试不用selenium直接使用phantomjs:

phantomjs --ignore-ssl-errors=true --ssl-protocol=any --web-security=false 1.js1.js有以下内容:

var webPage = require('webpage');
var page = webPage.create();
console.log('test666');
page.open('https://www.zoomeye.org/search/advanced', function (status) {
  var content = page.content;
  console.log('Content: ' + content);
  phantom.exit();
});

但是，它又不起作用了，可能是phantomjs的bug:(

Answer 1

浏览以下网址。它将为您提供将 https 与 phantomjs 一起使用所需的功能:-

http://phantomjs.org/api/command-line.html

您需要的主要能力如下:-

• --web-security=[true|false] 启用网络安全并禁止跨域 XHR(默认为 true)。还接受:[是|否]。
• --ssl-protocol=[sslv3|sslv2|tlsv1|tlsv1.1|tlsv1.2|any'] 设置安全连接的 SSL 协议(protocol)(默认为 SSLv3)。并非所有值可能受支持，具体取决于系统 OpenSSL 库。
• --ignore-ssl-errors=[true|false] 忽略 SSL 错误，例如过期或自签名证书错误(默认为 false)。还接受:[是|否]。

我有一个适合我的 java 代码:-

private static Capabilities getPhantomCapabilities(String OS) {
    DesiredCapabilities capabilities = null;
    ArrayList<String> cliArgsCap = new ArrayList<String>();
    capabilities = DesiredCapabilities.phantomjs();
    cliArgsCap.add("--web-security=false");
    cliArgsCap.add("--ssl-protocol=any");
    cliArgsCap.add("--ignore-ssl-errors=true");
    capabilities.setCapability("takesScreenshot", true);
    capabilities.setJavascriptEnabled(true);
    capabilities.setCapability(
        PhantomJSDriverService.PHANTOMJS_CLI_ARGS, cliArgsCap);
    capabilities.setCapability(
        PhantomJSDriverService.PHANTOMJS_GHOSTDRIVER_CLI_ARGS,
            new String[] { "--logLevel=2" });
    return capabilities;
}

在 python 中转换您的代码。它应该工作

希望对你有帮助:)