c - libcurl easy handle 'CURLOPT_RANGE' 选项不起作用

Question

我正在开发一个下载器，它将一个文件分成多个部分(每个部分都使用异步连接下载)，然后将这些部分合并到一个文件中。为此，我使用了 libcurl 库。

在“multi_handle2.c”的第 96 行，我告诉 libcurl 只下载一个字节范围，但它不遵守给定的范围，因为它正在下载整个文件。这是代码:

multi_handle2.c

#include <stdio.h>
#include <curl/curl.h>

#include "curl_extensions.h"

#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL     1

struct myprogress {
  double lastruntime;
  CURL *curl;
};

size_t write_data (void *ptr, size_t size, size_t nmemb, FILE *stream)
{
  size_t written = fwrite (ptr, size, nmemb, stream);
  return written;
}

static int xferinfo (void *p,
              curl_off_t dltotal,
              curl_off_t dlnow,
              curl_off_t ultotal,
              curl_off_t ulnow)
{
  struct myprogress *myp = (struct myprogress *)p;
  CURL *curl = myp->curl;
  double curtime = 0;

  curl_easy_getinfo (curl, CURLINFO_TOTAL_TIME, &curtime);

  if ((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL) {
    myp->lastruntime = curtime;
    fprintf (stderr, "TOTAL TIME: %f \r\n", curtime);
  }

  fprintf (stderr, "UP: %" CURL_FORMAT_CURL_OFF_T " of %" CURL_FORMAT_CURL_OFF_T
    " DOWN %" CURL_FORMAT_CURL_OFF_T " of %" CURL_FORMAT_CURL_OFF_T
    "\r\n",
    ulnow, ultotal, dlnow, dltotal);

  return 0;
}

void get_range(const int index, const int proc_number, const int total, int *begin, int *end)
{
  int range = total / proc_number;
  *begin = index*range;
  *end = *begin+range-1;
  if (index+1 == proc_number) {
    if (total % proc_number != 0) {
      *end += (total % proc_number);
    }
  }
}

int main()
{
  const int PROC_NUMBER = 8;

  CURLM *curlm;
  CURLMcode res;
  int still_running;
  CURL *handle[PROC_NUMBER];
  FILE *fp[PROC_NUMBER];
  struct myprogress prog;
  struct finfo_info info;
  int index;

  char *url = "https://codeload.github.com/GNOME/meld/zip/master";

  get_finfo(url, &info);
  //printf ("size: %d\n", info.size);

  curlm = curl_multi_init();

  for (index = 0; index < PROC_NUMBER; index++) {
    char filename[15];
    sprintf (filename, "part%d", index);

    fp[index] = fopen(filename, "wb");
    handle[index] = curl_easy_init();

    curl_easy_setopt(handle[index], CURLOPT_URL, url);
    curl_easy_setopt(handle[index], CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(handle[index], CURLOPT_WRITEDATA, fp[index]);

    curl_easy_setopt(handle[index], CURLOPT_XFERINFOFUNCTION, xferinfo);
    curl_easy_setopt(handle[index], CURLOPT_XFERINFODATA, &prog);
    curl_easy_setopt(handle[index], CURLOPT_NOPROGRESS, 0L);

    int begin, end;
    char *range = (char *) malloc(sizeof(char*));
    get_range(index, PROC_NUMBER, (int)info.size, &begin, &end);
    sprintf (range, "%d-%d", begin, end);
    printf ("%s\n", range);
    curl_easy_setopt(handle[index], CURLOPT_RANGE, range);

    curl_multi_add_handle(curlm, handle[index]);
  }

  res = curl_multi_perform(curlm, &still_running);

  /*******************/
  do {
    struct timeval timeout;
    int rc; /* select() return code */
    CURLMcode mc; /* curl_multi_fdset() return code */

    fd_set fdread;
    fd_set fdwrite;
    fd_set fdexcep;
    int maxfd = -1;

    long curl_timeo = -1;

    FD_ZERO(&fdread);
    FD_ZERO(&fdwrite);
    FD_ZERO(&fdexcep);

    /* set a suitable timeout to play around with */
    timeout.tv_sec = 1;
    timeout.tv_usec = 0;

    curl_multi_timeout(curlm, &curl_timeo);
    if (curl_timeo >= 0) {
      timeout.tv_sec = curl_timeo / 1000;
      if(timeout.tv_sec > 1)
        timeout.tv_sec = 1;
      else
        timeout.tv_usec = (curl_timeo % 1000) * 1000;
    }

    mc = curl_multi_fdset(curlm, &fdread, &fdwrite, &fdexcep, &maxfd);

    if(mc != CURLM_OK)
    {
      fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
      break;
    }

    if(maxfd == -1) {
    #ifdef _WIN32
      Sleep(100);
      rc = 0;
    #else
      /* Portable sleep for platforms other than Windows. */
      struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
      rc = select(0, NULL, NULL, NULL, &wait);
    #endif
    }
    else {
      /* Note that on some platforms 'timeout' may be modified by select().
         If you need access to the original value save a copy beforehand. */
      rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
    }

    switch(rc) {
    case -1:
      /* select error */
      break;
    case 0:
    default:
      /* timeout or readable/writable sockets */
      curl_multi_perform(curlm, &still_running);
      break;
    }
  } while(still_running);
  /*******************/

  if (res != CURLM_OK)
    printf ("failed\n");

  /* finals */
  for (index = 0; index < PROC_NUMBER; index++) {
    curl_multi_remove_handle(curlm, handle[index]);
    curl_easy_cleanup(handle[index]);
    fclose(fp[index]);
  }

  return 0;
}

curl_extensions.c

#include "curl_extensions.h"

#include <stdio.h>
#include <curl/curl.h>

static size_t finfo_memory_callback (void *contents,
                                     size_t size,
                                     size_t nmemb,
                                     void *userp)
{
  size_t realsize = size * nmemb;
  struct finfo_memory *mem = (struct finfo_memory *)userp;

  mem->memory = realloc(mem->memory, mem->size + realsize + 1);
  if(mem->memory == NULL) {
    /* out of memory! */
    printf("not enough memory (realloc returned NULL)\n");
    return 0;
  }

  memcpy(&(mem->memory[mem->size]), contents, realsize);
  mem->size += realsize;
  mem->memory[mem->size] = 0;

  if (DEBUG) {
    printf ("size: %d\n", realsize);
  }
  return realsize;
}

static int finfo_xferinfo (void *p,
                           curl_off_t dltotal,
                           curl_off_t dlnow,
                           curl_off_t ultotal,
                           curl_off_t ulnow)
{
  struct finfo_progress *myp = (struct finfo_progress *)p;
  myp->dltotal = dltotal;
  if (DEBUG) {
    printf ("getting size: %d\n", dltotal);
  }
  return (int)dltotal;
}

void get_finfo(char *url,
          struct finfo_info *pointer)
{
  CURL *curl;
  CURLcode res;
  struct finfo_progress prog;
  struct finfo_memory chunk;

  chunk.memory = malloc(1);
  chunk.size = 0;

  curl = curl_easy_init();
  if (curl) {
    curl_easy_setopt(curl, CURLOPT_URL, url);

    curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, finfo_xferinfo);
    curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &prog);
    curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);

    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, finfo_memory_callback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk);

    res = curl_easy_perform(curl);
    if (res != CURLE_OK)
      fprintf (stderr, "%s\n", curl_easy_strerror(res));

    curl_easy_cleanup(curl);
  }

  pointer->size = (size_t) prog.dltotal;

  //return !(res != CURLE_OK);
}

//int main()
//{
//  struct finfo_info info;
//  get_finfo ("https://raw.githubusercontent.com/bagder/curl/master/docs/examples/10-at-a-time.c", &info);
//  printf ("size: %d\n", info.size);
//
//  get_finfo ("http://download.cdn.mozilla.net/pub/firefox/releases/39.0/linux-i686/pt-BR/firefox-39.0.tar.bz2", &info);
//  printf ("size: %d\n", info.size);
//}

curl_extensions.h

#ifndef _CURL_EXTENSIONS_H
#define _CURL_EXTENSIONS_H

#include <curl/curl.h>

#define DEBUG 0

struct finfo_progress {
  curl_off_t dltotal;
};

struct finfo_memory {
  char *memory;
  size_t size;
};

struct finfo_info {
  size_t size;
};

static size_t finfo_memory_callback (void *contents, size_t size, size_t nmemb, void *userp);

static int finfo_xferinfo (void *p,
                           curl_off_t dltotal,
                           curl_off_t dlnow,
                           curl_off_t ultotal,
                           curl_off_t ulnow);

int finfo(char *url);

#endif /* _CURL_EXTENSIONS_H */

编译:

gcc multi_handle2.c -I curl_extensions.h curl_extensions.c -o multi_handle2 -lcurl

Answer 1

HTTP 服务器并没有被强制遵守 Range header ，它只是有时/经常被服务器忽略的来自客户端的请求。因此，您可能只是在询问不想玩 Range 的服务器，或者您只是在询问服务器可以提供其中一部分的特定 URL。如果输出是动态生成的，则通常是后者。

RFC 7233 section 3.1说:

A server MAY ignore the Range header field.

(CURLOPT_RANGE 手册页现已更新以提及这一事实。)