java - JAVA 中 HttpURLConnection 的响应产生垃圾值

Question

场景:我需要访问 HTTPS URL 来获取 .csv 文件或其内容。我可以在 Chrome 浏览器中使用高级 REST 客户端插件成功获取所需的内容。但是在尝试使用 java 代码执行相同操作时会得到一个垃圾值。

代码:

public class AnotherBean {
    private final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36";

    public void printAnotherMessage() {
        System.out.println("I am called by Quartz jobBean using CronTriggerFactoryBean");

        AnotherBean anotherBean = new AnotherBean();

        try {
            System.out.println("Testing 1 - Send Http GET request");
            anotherBean.sendGet();
            // System.out.println("\nTesting 2 - Send Http POST request");
            // anotherBean.sendPost();
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

    // HTTP GET request
    private void sendGet() throws Exception {
        String url = "https://xxx.xxx.com/xxxx/DownloadReport/2015/10/29/xxx/TransactionDetailReport.csv";
        URL obj = new URL(url);
        HttpURLConnection con = (HttpURLConnection) obj.openConnection();

        // optional default is GET
        con.setRequestMethod("GET");

        // add request header
        con.setRequestProperty("User-Agent", USER_AGENT);
        con.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        con.setRequestProperty("Accept-Language", "en-US,en;q=0.8");
        con.setRequestProperty("Content-Type", "text/plain; charset=utf-8");
        con.setRequestProperty("Accept-Encoding", "gzip, deflate, sdch");

        String urlParameters = "username=xxxx&password=xxxx";

        con.setDoOutput(true);
        con.setDoInput(true);

        OutputStream wr = con.getOutputStream();
        // wr.writeBytes(urlParameters);
        wr.write(urlParameters.getBytes("UTF-8"));
        wr.flush();
        wr.close();
        // con.connect();

        int responseCode = con.getResponseCode();
        System.out.println("\nSending 'GET' request to URL : " + url);
        System.out.println("Response Code : " + responseCode);
        System.out.println("Post parameters : " + urlParameters);

        BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
        String inputLine;
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
        }
        in.close();

        // print result
        System.out.println("Response: " + response.toString());
        System.out
                .println("Content Encoding: " + con.getContentEncoding() + "\nContent Length: " + con.getContentLength()
                        + "\nContent Type: " + con.getContentType() + "\nContent: " + con.getContent());
    }

    // HTTP POST request
    private void sendPost() throws Exception {
        String url = "https://xxx.xxx.com/xxxx/DownloadReport/2015/10/29/xxx/TransactionDetailReport.csv";
        URL obj = new URL(url);
        HttpsURLConnection con = (HttpsURLConnection) obj.openConnection();

        // add reuqest header
        con.setRequestMethod("POST");
        con.setRequestProperty("User-Agent", USER_AGENT);
        con.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        con.setRequestProperty("Accept-Language", "en-US,en;q=0.8");
        con.setRequestProperty("Content-Type", "text/plain; charset=utf-8");
        con.setRequestProperty("Accept-Encoding", "gzip, deflate, sdch");

        String urlParameters = "username=xxx&password=xxxx";
        // String urlParameters = "{" +
        // "\"username\":\"xxxx\",\"password\":\"xxxx\"" +
        // "}";

        // Send post request
        con.setDoOutput(true);
        con.setDoInput(true);
        // DataOutputStream wr = new DataOutputStream(con.getOutputStream());
        OutputStream wr = con.getOutputStream();
        // wr.writeBytes(urlParameters);
        wr.write(urlParameters.getBytes("UTF-8"));
        wr.flush();
        wr.close();
        // con.connect();
        int responseCode = con.getResponseCode();
        System.out.println("\nSending 'POST' request to URL : " + url);
        System.out.println("Post parameters : " + urlParameters);
        System.out.println("Response Code : " + responseCode);

        BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
        // BufferedReader inputReader = new BufferedReader(new
        // InputStreamReader((InputStream) con.getContent()));
        // String theString = IOUtils.toString(con.getInputStream(),
        // StandardCharsets.UTF_16BE);
        // byte[] byteArray = IOUtils.toByteArray(con.getInputStream());
        // String theString = IOUtils.toString(byteArray);
        String inputLine;
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
            // System.out.println(" " + inputLine);
        }

        /*
         * do { inputLine = inputReader.readLine(); response.append(inputLine +
         * "\n"); } while (inputLine != null);
         */

        // String finalResponse = new String(response, StandardCharsets.UTF_8);
        // inputReader.close();

        // print result
        System.out.println("Response: " + response.toString());
        // System.out.println("Response - theString: " + theString);
        // System.out.println("Response String: " + byteArray);
        System.out
                .println("Content Encoding: " + con.getContentEncoding() + "\nContent Length: " + con.getContentLength()
                        + "\nContent Type: " + con.getContentType() + "\nContent: " + con.getContent());
    }
}


**Response:**
***Sending 'GET' request to URL :*** "https://xxx.xxx.com/xxxx/DownloadReport/2015/10/29/xxx/TransactionDetailReport.csv";
***Response Code :*** 200
***Post parameters :*** username=xxxx&password=xxxx
***Response:*** ‹ •UmoÓ0þÞ_aR!©YÓn¬Í"¡2H›?ø„.ñ%1sâ`;[â¿syiצÍ`NU;ñùîžçÎwþ‹wŸ—·ß¾\³Ôf2ø› ?F÷ÂJfžÇ\vƒú5»ÖZiÜì4RZ`QÚ ½rJ»oœvËØõF¬•ö{ûZ?¢»D«2çs–«™ÆÁ2i%%óè^×c±=÷g°]†Š¯;3ЉÈçlòºXÑéɬX-öb•[7†LÈõœ9ŸÊHp`4ä?»…Te0boµ ......
***Content Encoding:*** gzip
***Content Length:*** 761
***Content Type:*** text/html
***Content:*** sun.net.www.protocol.http.HttpURLConnection$HttpInputStream@7c4ccb2f

即使使用 POST 方法，我也得到相同的响应。谁能告诉我哪里错了或者需要纠正什么。

提前致谢。

更新:根据 @Hendrik Simon 的回复，我可以得到可读格式的回复。但现在我面临另一个障碍。现在的响应是

Response: <!DOCTYPE html><html><head>    <title>500 - Server Error</title>    <meta charset="utf-8">    <style>        html {            background: none repeat scroll 0 0 #EEEEEE;        }        body {            margin: 17px 0 15px;            font-family: "Lucida Grande", Tahoma, Arial, Verdana, sans-serif;            padding: 0;            text-align: center;            font-size: small;        }        #content {            display: block;            margin: 20px 14px 10px;            padding: 0 0 20px;            position: relative;        }        #container {            background: none repeat scroll 0 0 #FFFFFF;            border: 1px solid #CCCCCC;            box-shadow: 0 0 5px rgba(0, 0, 0, 0.2);            text-align: left;            margin: 0 auto;            width: 640px;            min-height: 200px;        }        h2 {            font-size: 160%;            clear: left;            font-weight: 400;            margin: 0 0 5px;            text-align: left;        }        #content div {            color: #555555;            font-size: 95%;            line-height: 135%;        }        #logo {            background-color: #003C69;            background-image: none;            margin: 0;            min-height: 0;            padding: 0;            text-decoration: none;            text-indent: -9000px;        }        #logo a {            background-repeat: no-repeat;            background-image: url("cybersource_logo.gif");            height: 55px;            min-height: 0;            display: block;            overflow: hidden;        }    </style></head><body><div id="container">    <h1 id="logo"><a>CyberSource</a></h1>    <div id="content">        <h2>The server encountered an error and could not complete your request.</h2>        <br/>        <div>If the problem persists please report your problem.</div>    </div></div><script src="ntpagetag.js"></script></body></html>

当我在浏览器或 REST 客户端上使用相同的 URL 时，通常会出现一个弹出窗口来输入凭据。成功提交后，我得到了所需的数据。谁能指导我如何使用 JAVA 代码处理这种情况？

Answer 1

似乎响应是 gzip 编码的，并且您没有解压缩它。

尝试删除以下行

    con.setRequestProperty("Accept-Encoding", "gzip, deflate, sdch");

更新:

正如 @RealSkeptic 正确提到的，如果它是 GET 请求，则无法将数据写入连接。对于 GET 请求，您必须将参数添加到 URL，如下所示:

http://...?param1=bla&param2=blubb

我认为，只要您调用 con.setDoOutput(true); 您的连接就会自动执行 POST 请求，无论您设置什么请求方法。因此，在您的示例中，您永远不会发出 GET 请求。

例如，当您使用 Wireshark 分析流量时，您可以轻松看到这一点。

对于服务器错误，如果不重现它，就很难分析它。在不知道 URL 的情况下重新生成它是不可能的，至少对我来说是这样。