go - 为什么Golang不能下载某些网页？

Question

我想下载 Fantasy Football Data 以在 Go 中进行分析，但是当我尝试从 this api page 下载时然后我得到一个空的响应，即使代码适用于其他网站，例如this api page

最小复制，输出一个空数组。

package main

import (
    "fmt"
    "io/ioutil"
    "net/http"
    "time"
)

const AllPlayerData = "https://fantasy.premierleague.com/drf/bootstrap-static"

func main() {
    downloadAllData()
}

func downloadAllData() {
    client := &http.Client{
        Timeout: 20 * time.Second,
    }

    response, err := client.Get(AllPlayerData)
    if err != nil {
        fmt.Println("Unable to download player data.")
        return
    }

    body, err := ioutil.ReadAll(response.Body)
    if err != nil {
        fmt.Println("Failed to read response")
        return
    }

    defer response.Body.Close()

    fmt.Println(body)
}

同一网页在 Python 中下载正常:

import requests
url = "https://fantasy.premierleague.com/drf/bootstrap-static"
r = requests.get(url)
print(r.content)

我认为这与例如Ajax calls因为在 Chrome 中查看网络请求不会显示任何超出页面加载本身的内容

Answer 1

他们正在对用户代理进行某种验证，以下代码有效:

package main

import (
    "fmt"
    "io/ioutil"
    "net/http"
    "time"
)

const AllPlayerData = "https://fantasy.premierleague.com/drf/bootstrap-static"

func main() {
    downloadAllData()
}

func downloadAllData() {
    client := &http.Client{
        Timeout: 20 * time.Second,
    }

    request, err := http.NewRequest(http.MethodGet, AllPlayerData, nil)
    if err != nil {
        fmt.Println("Unable to create request.")
        return
    }
    request.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36")
    response, err := client.Do(request)
    if err != nil {
        fmt.Println("Unable to download player data.")
        return
    }

    body, err := ioutil.ReadAll(response.Body)
    if err != nil {
        fmt.Println("Failed to read response")
        return
    }

    defer response.Body.Close()

    fmt.Println(string(body))
}