csv - 从新的雅虎财经历史数据下载 csv 时出错

Question

既然旧的 API 已经正式失效，我已经尝试让一些代码适用于新的雅虎财务历史数据。代码如下:

Private Function DownloadData(ByVal StockSymbol As String)

    Try

        'https://query1.finance.yahoo.com/v7/finance/download/CPG.TO?period1=1492824351&period2=1495416351&interval=1d&events=history&crumb=M3Ig5MvcK77

        Dim Period1 As Integer = (New DateTime(2007, 1, 1, 0, 0, 0) - New DateTime(1980, 1, 1, 0, 0, 0)).TotalSeconds
        Dim Period2 As Integer = (DateTime.UtcNow - New DateTime(1980, 1, 1, 0, 0, 0)).TotalSeconds

        Dim csvData As String = String.Empty

        Using Web As New WebClient

            Dim strm As Stream = Web.OpenRead("https://finance.yahoo.com/quote/" & StockSymbol & "/history?p=" & StockSymbol)

            Dim Crumb As String = ScrapeCrumb(strm)

            Dim s As String = "https://query1.finance.yahoo.com/v7/finance/download/" & StockSymbol &
                                "?period1=" & Period1.ToString & "&period2=" & Period2.ToString & "&interval=1d&events=history&crumb=" & Crumb
            csvData = Web.DownloadString(s)

        End Using

        Return csvData

    Catch ex As Exception

        Return String.Empty

    End Try

End Function
Private Function ScrapeCrumb(ByVal strm As Stream) As String

    Try

        Dim Output As String = String.Empty

        Using sr As New StreamReader(strm)
            Output = sr.ReadToEnd()
            ' Close and clean up the StreamReader
            sr.Close()
        End Using

        Dim Result As String = Output.Remove(0, Output.IndexOf("CrumbStore"))
        Result = Result.Remove(0, 22)
        Result = Result.Remove(Result.IndexOf("}"), Result.Length - Result.IndexOf("}"))
        Result = Result.Remove(Result.Length - 1, 1)

        Return Result

    Catch ex As Exception

        Return String.Empty

    End Try

End Function

但我不断收到错误

The remote server returned an error: (401) Unauthorized.

我觉得这与 cookies/crumb 交易有关，但我正在抓取它，所以我不确定为什么这不授予下载文件的权限。有人有什么想法吗？

编辑:使用vb语言

Answer 1

您收到错误“(401) Unauthorized”，因为您没有有效的 Cookie(Cookie“B”)值。

我设法编写了一个 .NET 类来从雅虎财经获取有效 token (cookie 和 crumb)

有关从新雅虎财经获取历史数据的完整 API 库，您可以访问 YahooFinanceAPI在 Github

这是获取 cookie 和面包屑的类

Token.cs

using System;
using System.Diagnostics;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;

namespace YahooFinanceAPI
{
    /// <summary>
    /// Class for fetching token (cookie and crumb) from Yahoo Finance
    /// Copyright Dennis Lee
    /// 19 May 2017
    /// 
    /// </summary>
    public class Token
    {
        public static string Cookie { get; set; }
        public static string Crumb { get; set; }

        private static Regex regex_crumb;
        /// <summary>
        /// Refresh cookie and crumb value Yahoo Fianance
        /// </summary>
        /// <param name="symbol">Stock ticker symbol</param>
        /// <returns></returns>
        public static bool Refresh(string symbol = "SPY")
        {

            try
            {
                Token.Cookie = "";
                Token.Crumb = "";

                string url_scrape = "https://finance.yahoo.com/quote/{0}?p={0}";
                //url_scrape = "https://finance.yahoo.com/quote/{0}/history"

                string url = string.Format(url_scrape, symbol);

                HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);

                request.CookieContainer = new CookieContainer();
                request.Method = "GET";

                using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
                {

                    string cookie = response.GetResponseHeader("Set-Cookie").Split(';')[0];

                    string html = "";

                    using (Stream stream = response.GetResponseStream())
                    {
                        html = new StreamReader(stream).ReadToEnd();
                    }

                    if (html.Length < 5000)
                        return false;
                    string crumb = getCrumb(html);
                    html = "";

                    if (crumb != null)
                    {
                        Token.Cookie = cookie;
                        Token.Crumb = crumb;
                        Debug.Print("Crumb: '{0}', Cookie: '{1}'", crumb, cookie);
                        return true;
                    }

                }

            }
            catch (Exception ex)
            {
                Debug.Print(ex.Message);
            }

            return false;

        }

        /// <summary>
        /// Get crumb value from HTML
        /// </summary>
        /// <param name="html">HTML code</param>
        /// <returns></returns>
        private static string getCrumb(string html)
        {

            string crumb = null;

            try
            {
                //initialize on first time use
                if (regex_crumb == null)
                    regex_crumb = new Regex("CrumbStore\":{\"crumb\":\"(?<crumb>\\w+)\"}", 
                    RegexOptions.CultureInvariant | RegexOptions.Compiled, TimeSpan.FromSeconds(5));

                MatchCollection matches = regex_crumb.Matches(html);

                if (matches.Count > 0)
                {
                    crumb = matches[0].Groups["crumb"].Value;
                }
                else
                {
                    Debug.Print("Regex no match");
                }

                //prevent regex memory leak
                matches = null;

            }
            catch (Exception ex)
            {
                Debug.Print(ex.Message);
            }

            GC.Collect();
            return crumb;

        }

    }
}