libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
   9 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  10 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  11
  12 /*
  13 =================================================================
  14
  15   Minimal set of definitions from libcurl
  16
  17   WARNING: for a matter of simplicity, several pointer types are
  18   casted to "void*", and most enumerated values are not included
  19
  20 =================================================================
  21 */
  22
  23 typedef struct CURL_s CURL;
  24 typedef struct CURLM_s CURLM;
  25 typedef struct curl_slist curl_slist;
  26 typedef enum
  27 {
  28         CURLE_OK = 0
  29 }
  30 CURLcode;
  31 typedef enum
  32 {
  33         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  34         CURLM_OK = 0
  35 }
  36 CURLMcode;
  37 #define CURL_GLOBAL_NOTHING 0
  38 #define CURL_GLOBAL_SSL 1
  39 #define CURL_GLOBAL_WIN32 2
  40 #define CURLOPTTYPE_LONG          0
  41 #define CURLOPTTYPE_OBJECTPOINT   10000
  42 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  43 #define CURLOPTTYPE_OFF_T         30000
  44 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  45 typedef enum
  46 {
  47         CINIT(WRITEDATA, OBJECTPOINT, 1),
  48         CINIT(URL,  OBJECTPOINT, 2),
  49         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  50         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  51         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  52         CINIT(REFERER, OBJECTPOINT, 16),
  53         CINIT(USERAGENT, OBJECTPOINT, 18),
  54         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  55         CINIT(LOW_SPEED_TIME, LONG, 20),
  56         CINIT(RESUME_FROM, LONG, 21),
  57         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  58         CINIT(POST, LONG, 47),         /* HTTP POST method */
  59         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  60         CINIT(POSTFIELDSIZE, LONG, 60),
  61         CINIT(PRIVATE, OBJECTPOINT, 103),
  62         CINIT(PROTOCOLS, LONG, 181),
  63         CINIT(REDIR_PROTOCOLS, LONG, 182)
  64 }
  65 CURLoption;
  66 #define CURLPROTO_HTTP   (1<<0)
  67 #define CURLPROTO_HTTPS  (1<<1)
  68 #define CURLPROTO_FTP    (1<<2)
  69 typedef enum
  70 {
  71         CURLINFO_TEXT = 0,
  72         CURLINFO_HEADER_IN,    /* 1 */
  73         CURLINFO_HEADER_OUT,   /* 2 */
  74         CURLINFO_DATA_IN,      /* 3 */
  75         CURLINFO_DATA_OUT,     /* 4 */
  76         CURLINFO_SSL_DATA_IN,  /* 5 */
  77         CURLINFO_SSL_DATA_OUT, /* 6 */
  78         CURLINFO_END
  79 }
  80 curl_infotype;
  81 #define CURLINFO_STRING   0x100000
  82 #define CURLINFO_LONG     0x200000
  83 #define CURLINFO_DOUBLE   0x300000
  84 #define CURLINFO_SLIST    0x400000
  85 #define CURLINFO_MASK     0x0fffff
  86 #define CURLINFO_TYPEMASK 0xf00000
  87 typedef enum
  88 {
  89         CURLINFO_NONE, /* first, never use this */
  90         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  91         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  92         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  93         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  94         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  95         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  96         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  97         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  98         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  99         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 100         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 101         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 102         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 103         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 104         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 105         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 106         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 107         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 108         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 109         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 110         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 111         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 112         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 113         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 114         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 115         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 116         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 117 }
 118 CURLINFO;
 119
 120 typedef enum
 121 {
 122         CURLMSG_NONE, /* first, not used */
 123         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 124                                          the CURLcode of the transfer */
 125         CURLMSG_LAST
 126 }
 127 CURLMSG;
 128 typedef struct
 129 {
 130         CURLMSG msg;       /* what this message means */
 131         CURL *easy_handle; /* the handle it concerns */
 132         union
 133         {
 134                 void *whatever;    /* message-specific data */
 135                 CURLcode result;   /* return code for transfer */
 136         }
 137         data;
 138 }
 139 CURLMsg;
 140
 141 static void (*qcurl_global_init) (long flags);
 142 static void (*qcurl_global_cleanup) (void);
 143
 144 static CURL * (*qcurl_easy_init) (void);
 145 static void (*qcurl_easy_cleanup) (CURL *handle);
 146 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 147 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 148 static const char * (*qcurl_easy_strerror) (CURLcode);
 149
 150 static CURLM * (*qcurl_multi_init) (void);
 151 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 152 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 153 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 154 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 155 static void (*qcurl_multi_cleanup) (CURLM *);
 156 static const char * (*qcurl_multi_strerror) (CURLcode);
 157 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 158 static void (*qcurl_slist_free_all) (curl_slist *list);
 159
 160 static dllfunction_t curlfuncs[] =
 161 {
 162         {"curl_global_init",            (void **) &qcurl_global_init},
 163         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 164         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 165         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 166         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 167         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 168         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 169         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 170         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 171         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 172         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 173         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 174         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 175         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 176         {"curl_slist_append",           (void **) &qcurl_slist_append},
 177         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 178         {NULL, NULL}
 179 };
 180
 181 // Handle for CURL DLL
 182 static dllhandle_t curl_dll = NULL;
 183 // will be checked at many places to find out if qcurl calls are allowed
 184
 185 typedef struct downloadinfo_s
 186 {
 187         char filename[MAX_OSPATH];
 188         char url[1024];
 189         char referer[256];
 190         qfile_t *stream;
 191         fs_offset_t startpos;
 192         CURL *curle;
 193         qboolean started;
 194         qboolean ispak;
 195         unsigned long bytes_received; // for buffer
 196         double bytes_received_curl; // for throttling
 197         double bytes_sent_curl; // for throttling
 198         struct downloadinfo_s *next, *prev;
 199         qboolean forthismap;
 200         double maxspeed;
 201         curl_slist *slist; // http headers
 202
 203         unsigned char *buffer;
 204         size_t buffersize;
 205         curl_callback_t callback;
 206         void *callback_data;
 207
 208         const unsigned char *postbuf;
 209         size_t postbufsize;
 210         const char *post_content_type;
 211         const char *extraheaders;
 212 }
 213 downloadinfo;
 214 static downloadinfo *downloads = NULL;
 215 static int numdownloads = 0;
 216
 217 static qboolean noclear = FALSE;
 218
 219 static int numdownloads_fail = 0;
 220 static int numdownloads_success = 0;
 221 static int numdownloads_added = 0;
 222 static char command_when_done[256] = "";
 223 static char command_when_error[256] = "";
 224
 225 /*
 226 ====================
 227 Curl_CommandWhenDone
 228
 229 Sets the command which is to be executed when the last download completes AND
 230 all downloads since last server connect ended with a successful status.
 231 Setting the command to NULL clears it.
 232 ====================
 233 */
 234 void Curl_CommandWhenDone(const char *cmd)
 235 {
 236         if(!curl_dll)
 237                 return;
 238         if(cmd)
 239                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 240         else
 241                 *command_when_done = 0;
 242 }
 243
 244 /*
 245 FIXME
 246 Do not use yet. Not complete.
 247 Problem: what counts as an error?
 248 */
 249
 250 void Curl_CommandWhenError(const char *cmd)
 251 {
 252         if(!curl_dll)
 253                 return;
 254         if(cmd)
 255                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 256         else
 257                 *command_when_error = 0;
 258 }
 259
 260 /*
 261 ====================
 262 Curl_Clear_forthismap
 263
 264 Clears the "will disconnect on failure" flags.
 265 ====================
 266 */
 267 void Curl_Clear_forthismap(void)
 268 {
 269         downloadinfo *di;
 270         if(noclear)
 271                 return;
 272         for(di = downloads; di; di = di->next)
 273                 di->forthismap = false;
 274         Curl_CommandWhenError(NULL);
 275         Curl_CommandWhenDone(NULL);
 276         numdownloads_fail = 0;
 277         numdownloads_success = 0;
 278         numdownloads_added = 0;
 279 }
 280
 281 /*
 282 ====================
 283 Curl_Have_forthismap
 284
 285 Returns true if a download needed for the current game is running.
 286 ====================
 287 */
 288 qboolean Curl_Have_forthismap(void)
 289 {
 290         return numdownloads_added != 0;
 291 }
 292
 293 void Curl_Register_predownload(void)
 294 {
 295         Curl_CommandWhenDone("cl_begindownloads");
 296         Curl_CommandWhenError("cl_begindownloads");
 297 }
 298
 299 /*
 300 ====================
 301 Curl_CheckCommandWhenDone
 302
 303 Checks if a "done command" is to be executed.
 304 All downloads finished, at least one success since connect, no single failure
 305 -> execute the command.
 306 */
 307 static void Curl_CheckCommandWhenDone(void)
 308 {
 309         if(!curl_dll)
 310                 return;
 311         if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
 312         {
 313                 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 314                 Cbuf_AddText("\n");
 315                 Cbuf_AddText(command_when_done);
 316                 Cbuf_AddText("\n");
 317                 Curl_Clear_forthismap();
 318         }
 319         else if(numdownloads_added && numdownloads_fail && *command_when_error)
 320         {
 321                 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 322                 Cbuf_AddText("\n");
 323                 Cbuf_AddText(command_when_error);
 324                 Cbuf_AddText("\n");
 325                 Curl_Clear_forthismap();
 326         }
 327 }
 328
 329 /*
 330 ====================
 331 CURL_CloseLibrary
 332
 333 Load the cURL DLL
 334 ====================
 335 */
 336 static qboolean CURL_OpenLibrary (void)
 337 {
 338         const char* dllnames [] =
 339         {
 340 #if defined(WIN32)
 341                 "libcurl-4.dll",
 342                 "libcurl-3.dll",
 343 #elif defined(MACOSX)
 344                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 345                 "libcurl.3.dylib", // Mac OS X Tiger
 346                 "libcurl.2.dylib", // Mac OS X Panther
 347 #else
 348                 "libcurl.so.4",
 349                 "libcurl.so.3",
 350                 "libcurl.so", // FreeBSD
 351 #endif
 352                 NULL
 353         };
 354
 355         // Already loaded?
 356         if (curl_dll)
 357                 return true;
 358
 359         // Load the DLL
 360         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
 361 }
 362
 363
 364 /*
 365 ====================
 366 CURL_CloseLibrary
 367
 368 Unload the cURL DLL
 369 ====================
 370 */
 371 static void CURL_CloseLibrary (void)
 372 {
 373         Sys_UnloadLibrary (&curl_dll);
 374 }
 375
 376
 377 static CURLM *curlm = NULL;
 378 static double bytes_received = 0; // used for bandwidth throttling
 379 static double bytes_sent = 0; // used for bandwidth throttling
 380 static double curltime = 0;
 381
 382 /*
 383 ====================
 384 CURL_fwrite
 385
 386 fwrite-compatible function that writes the data to a file. libcurl can call
 387 this.
 388 ====================
 389 */
 390 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 391 {
 392         fs_offset_t ret = -1;
 393         size_t bytes = size * nmemb;
 394         downloadinfo *di = (downloadinfo *) vdi;
 395
 396         if(di->buffer)
 397         {
 398                 if(di->bytes_received + bytes <= di->buffersize)
 399                 {
 400                         memcpy(di->buffer + di->bytes_received, data, bytes);
 401                         ret = bytes;
 402                 }
 403                 // otherwise: buffer overrun, ret stays -1
 404         }
 405
 406         if(di->stream)
 407         {
 408                 ret = FS_Write(di->stream, data, bytes);
 409         }
 410
 411         di->bytes_received += bytes;
 412
 413         return ret; // why not ret / nmemb?
 414 }
 415
 416 typedef enum
 417 {
 418         CURL_DOWNLOAD_SUCCESS = 0,
 419         CURL_DOWNLOAD_FAILED,
 420         CURL_DOWNLOAD_ABORTED,
 421         CURL_DOWNLOAD_SERVERERROR
 422 }
 423 CurlStatus;
 424
 425 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 426 {
 427         downloadinfo *di = (downloadinfo *) cbdata;
 428         switch(status)
 429         {
 430                 case CURLCBSTATUS_OK:
 431                         Con_DPrintf("Download of %s: OK\n", di->filename);
 432                         break;
 433                 case CURLCBSTATUS_FAILED:
 434                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 435                         break;
 436                 case CURLCBSTATUS_ABORTED:
 437                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 438                         break;
 439                 case CURLCBSTATUS_SERVERERROR:
 440                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 441                         break;
 442                 case CURLCBSTATUS_UNKNOWN:
 443                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 444                         break;
 445                 default:
 446                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 447                         break;
 448         }
 449 }
 450
 451 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 452 {
 453         curl_default_callback(status, length_received, buffer, cbdata);
 454 }
 455
 456 /*
 457 ====================
 458 Curl_EndDownload
 459
 460 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 461 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 462 code from libcurl, or 0, if another error has occurred.
 463 ====================
 464 */
 465 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 466 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 467 {
 468         qboolean ok = false;
 469         if(!curl_dll)
 470                 return;
 471         switch(status)
 472         {
 473                 case CURL_DOWNLOAD_SUCCESS:
 474                         ok = true;
 475                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 476                         break;
 477                 case CURL_DOWNLOAD_FAILED:
 478                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 479                         break;
 480                 case CURL_DOWNLOAD_ABORTED:
 481                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 482                         break;
 483                 case CURL_DOWNLOAD_SERVERERROR:
 484                         // reopen to enforce it to have zero bytes again
 485                         if(di->stream)
 486                         {
 487                                 FS_Close(di->stream);
 488                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 489                         }
 490
 491                         if(di->callback)
 492                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 493                         break;
 494                 default:
 495                         if(di->callback)
 496                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 497                         break;
 498         }
 499
 500         if(di->curle)
 501         {
 502                 qcurl_multi_remove_handle(curlm, di->curle);
 503                 qcurl_easy_cleanup(di->curle);
 504                 if(di->slist)
 505                         qcurl_slist_free_all(di->slist);
 506         }
 507
 508         if(!di->callback && ok && !di->bytes_received)
 509         {
 510                 Con_Printf("ERROR: empty file\n");
 511                 ok = false;
 512         }
 513
 514         if(di->stream)
 515                 FS_Close(di->stream);
 516
 517         if(ok && di->ispak)
 518         {
 519                 ok = FS_AddPack(di->filename, NULL, true);
 520                 if(!ok)
 521                 {
 522                         // pack loading failed?
 523                         // this is critical
 524                         // better clear the file again...
 525                         di->stream = FS_OpenRealFile(di->filename, "wb", false);
 526                         FS_Close(di->stream);
 527
 528                         if(di->startpos && !di->callback)
 529                         {
 530                                 // this was a resume?
 531                                 // then try to redownload it without reporting the error
 532                                 Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->ispak, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL);
 533                                 di->forthismap = false; // don't count the error
 534                         }
 535                 }
 536         }
 537
 538         if(di->prev)
 539                 di->prev->next = di->next;
 540         else
 541                 downloads = di->next;
 542         if(di->next)
 543                 di->next->prev = di->prev;
 544
 545         --numdownloads;
 546         if(di->forthismap)
 547         {
 548                 if(ok)
 549                         ++numdownloads_success;
 550                 else
 551                         ++numdownloads_fail;
 552         }
 553         Z_Free(di);
 554 }
 555
 556 /*
 557 ====================
 558 CleanURL
 559
 560 Returns a "cleaned up" URL for display (to strip login data)
 561 ====================
 562 */
 563 static const char *CleanURL(const char *url)
 564 {
 565         static char urlbuf[1024];
 566         const char *p, *q, *r;
 567
 568         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 569         p = strstr(url, "://");
 570         if(p)
 571         {
 572                 q = strchr(p + 3, '@');
 573                 if(q)
 574                 {
 575                         r = strchr(p + 3, '/');
 576                         if(!r || q < r)
 577                         {
 578                                 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s%s", (int)(p - url + 3), url, q + 1);
 579                                 return urlbuf;
 580                         }
 581                 }
 582         }
 583
 584         return url;
 585 }
 586
 587 /*
 588 ====================
 589 CheckPendingDownloads
 590
 591 checks if there are free download slots to start new downloads in.
 592 To not start too many downloads at once, only one download is added at a time,
 593 up to a maximum number of cl_curl_maxdownloads are running.
 594 ====================
 595 */
 596 static void CheckPendingDownloads(void)
 597 {
 598         const char *h;
 599         if(!curl_dll)
 600                 return;
 601         if(numdownloads < cl_curl_maxdownloads.integer)
 602         {
 603                 downloadinfo *di;
 604                 for(di = downloads; di; di = di->next)
 605                 {
 606                         if(!di->started)
 607                         {
 608                                 if(!di->buffer)
 609                                 {
 610                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url), di->filename);
 611
 612                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 613                                         if(!di->stream)
 614                                         {
 615                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 616                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 617                                                 return;
 618                                         }
 619                                         FS_Seek(di->stream, 0, SEEK_END);
 620                                         di->startpos = FS_Tell(di->stream);
 621
 622                                         if(di->startpos > 0)
 623                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 624                                         Con_Print("...\n");
 625                                 }
 626                                 else
 627                                 {
 628                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url));
 629                                         di->startpos = 0;
 630                                 }
 631
 632                                 di->curle = qcurl_easy_init();
 633                                 di->slist = NULL;
 634                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 635                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 636                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 637                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 638                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 639                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 640                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 641                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 642                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 643                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 644                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 645                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 646                                 {
 647                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 648                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 649                                 }
 650                                 if(di->post_content_type)
 651                                 {
 652                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 653                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 654                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 655                                         di->slist = qcurl_slist_append(di->slist, va("Content-Type: %s", di->post_content_type));
 656                                 }
 657
 658                                 // parse extra headers into slist
 659                                 // \n separated list!
 660                                 h = di->extraheaders;
 661                                 while(h)
 662                                 {
 663                                         const char *hh = strchr(h, '\n');
 664                                         if(hh)
 665                                         {
 666                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 667                                                 memcpy(buf, h, hh - h);
 668                                                 buf[hh - h] = 0;
 669                                                 di->slist = qcurl_slist_append(di->slist, buf);
 670                                                 h = hh + 1;
 671                                         }
 672                                         else
 673                                         {
 674                                                 di->slist = qcurl_slist_append(di->slist, h);
 675                                                 h = NULL;
 676                                         }
 677                                 }
 678
 679                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 680
 681
 682                                 qcurl_multi_add_handle(curlm, di->curle);
 683                                 di->started = true;
 684                                 ++numdownloads;
 685                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 686                                         break;
 687                         }
 688                 }
 689         }
 690 }
 691
 692 /*
 693 ====================
 694 Curl_Init
 695
 696 this function MUST be called before using anything else in this file.
 697 On Win32, this must be called AFTER WSAStartup has been done!
 698 ====================
 699 */
 700 void Curl_Init(void)
 701 {
 702         CURL_OpenLibrary();
 703         if(!curl_dll)
 704                 return;
 705         qcurl_global_init(CURL_GLOBAL_NOTHING);
 706         curlm = qcurl_multi_init();
 707 }
 708
 709 /*
 710 ====================
 711 Curl_Shutdown
 712
 713 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 714 ====================
 715 */
 716 void Curl_ClearRequirements(void);
 717 void Curl_Shutdown(void)
 718 {
 719         if(!curl_dll)
 720                 return;
 721         Curl_ClearRequirements();
 722         Curl_CancelAll();
 723         CURL_CloseLibrary();
 724         curl_dll = NULL;
 725 }
 726
 727 /*
 728 ====================
 729 Curl_Find
 730
 731 Finds the internal information block for a download given by file name.
 732 ====================
 733 */
 734 static downloadinfo *Curl_Find(const char *filename)
 735 {
 736         downloadinfo *di;
 737         if(!curl_dll)
 738                 return NULL;
 739         for(di = downloads; di; di = di->next)
 740                 if(!strcasecmp(di->filename, filename))
 741                         return di;
 742         return NULL;
 743 }
 744
 745 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 746 {
 747         downloadinfo *di;
 748         if(!curl_dll)
 749                 return;
 750         for(di = downloads; di; )
 751         {
 752                 if(di->callback == callback && di->callback_data == cbdata)
 753                 {
 754                         di->callback = curl_quiet_callback; // do NOT call the callback
 755                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 756                         di = downloads;
 757                 }
 758                 else
 759                         di = di->next;
 760         }
 761 }
 762
 763 /*
 764 ====================
 765 Curl_Begin
 766
 767 Starts a download of a given URL to the file name portion of this URL (or name
 768 if given) in the "dlcache/" folder.
 769 ====================
 770 */
 771 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 772 {
 773         if(!curl_dll)
 774         {
 775                 return false;
 776         }
 777         else
 778         {
 779                 char fn[MAX_OSPATH];
 780                 char urlbuf[1024];
 781                 const char *p, *q;
 782                 size_t length;
 783                 downloadinfo *di;
 784
 785                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 786                 p = strchr(URL, ':');
 787                 if(p)
 788                 {
 789                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 790                         {
 791                                 char addressstring[128];
 792                                 *addressstring = 0;
 793                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 794                                 q = strchr(addressstring, ':');
 795                                 if(!q)
 796                                         q = addressstring + strlen(addressstring);
 797                                 if(*addressstring)
 798                                 {
 799                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 800                                         URL = urlbuf;
 801                                 }
 802                         }
 803                 }
 804
 805                 // Note: This extraction of the file name portion is NOT entirely correct.
 806                 //
 807                 // It does the following:
 808                 //
 809                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 810                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 811                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 812                 //
 813                 // However, I'd like to keep this "buggy" behavior so that PHP script
 814                 // authors can write download scripts without having to enable
 815                 // AcceptPathInfo on Apache. They just have to ensure that their script
 816                 // can be called with such a "fake" path name like
 817                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 818                 //
 819                 // By the way, such PHP scripts should either send the file or a
 820                 // "Location:" redirect; PHP code example:
 821                 //
 822                 //   header("Location: http://www.example.com/");
 823                 //
 824                 // By the way, this will set User-Agent to something like
 825                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 826                 // dp://serverhost:serverport/ so you can filter on this; an example
 827                 // httpd log file line might be:
 828                 //
 829                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 830
 831                 if(!name)
 832                         name = CleanURL(URL);
 833
 834                 if(!buf)
 835                 {
 836                         p = strrchr(name, '/');
 837                         p = p ? (p+1) : name;
 838                         q = strchr(p, '?');
 839                         length = q ? (size_t)(q - p) : strlen(p);
 840                         dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 841
 842                         name = fn; // make it point back
 843
 844                         // already downloading the file?
 845                         {
 846                                 downloadinfo *di = Curl_Find(fn);
 847                                 if(di)
 848                                 {
 849                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url));
 850
 851                                         // however, if it was not for this map yet...
 852                                         if(forthismap && !di->forthismap)
 853                                         {
 854                                                 di->forthismap = true;
 855                                                 // this "fakes" a download attempt so the client will wait for
 856                                                 // the download to finish and then reconnect
 857                                                 ++numdownloads_added;
 858                                         }
 859
 860                                         return false;
 861                                 }
 862                         }
 863
 864                         if(ispak && FS_FileExists(fn))
 865                         {
 866                                 qboolean already_loaded;
 867                                 if(FS_AddPack(fn, &already_loaded, true))
 868                                 {
 869                                         Con_DPrintf("%s already exists, not downloading!\n", fn);
 870                                         if(already_loaded)
 871                                                 Con_DPrintf("(pak was already loaded)\n");
 872                                         else
 873                                         {
 874                                                 if(forthismap)
 875                                                 {
 876                                                         ++numdownloads_added;
 877                                                         ++numdownloads_success;
 878                                                 }
 879                                         }
 880
 881                                         return false;
 882                                 }
 883                                 else
 884                                 {
 885                                         qfile_t *f = FS_OpenRealFile(fn, "rb", false);
 886                                         if(f)
 887                                         {
 888                                                 char buf[4] = {0};
 889                                                 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 890
 891                                                 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 892                                                 {
 893                                                         Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 894                                                         FS_Close(f);
 895                                                         f = FS_OpenRealFile(fn, "wb", false);
 896                                                         if(f)
 897                                                                 FS_Close(f);
 898                                                 }
 899                                                 else
 900                                                 {
 901                                                         // OK
 902                                                         FS_Close(f);
 903                                                 }
 904                                         }
 905                                 }
 906                         }
 907                 }
 908
 909                 // if we get here, we actually want to download... so first verify the
 910                 // URL scheme (so one can't read local files using file://)
 911                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
 912                 {
 913                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
 914                         return false;
 915                 }
 916
 917                 if(forthismap)
 918                         ++numdownloads_added;
 919                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 920                 strlcpy(di->filename, name, sizeof(di->filename));
 921                 strlcpy(di->url, URL, sizeof(di->url));
 922                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 923                 di->forthismap = forthismap;
 924                 di->stream = NULL;
 925                 di->startpos = 0;
 926                 di->curle = NULL;
 927                 di->started = false;
 928                 di->ispak = (ispak && !buf);
 929                 di->maxspeed = maxspeed;
 930                 di->bytes_received = 0;
 931                 di->bytes_received_curl = 0;
 932                 di->bytes_sent_curl = 0;
 933                 di->extraheaders = extraheaders;
 934                 di->next = downloads;
 935                 di->prev = NULL;
 936                 if(di->next)
 937                         di->next->prev = di;
 938
 939                 di->buffer = buf;
 940                 di->buffersize = bufsize;
 941                 if(callback == NULL)
 942                 {
 943                         di->callback = curl_default_callback;
 944                         di->callback_data = di;
 945                 }
 946                 else
 947                 {
 948                         di->callback = callback;
 949                         di->callback_data = cbdata;
 950                 }
 951
 952                 if(post_content_type)
 953                 {
 954                         di->post_content_type = post_content_type;
 955                         di->postbuf = postbuf;
 956                         di->postbufsize = postbufsize;
 957                 }
 958                 else
 959                 {
 960                         di->post_content_type = NULL;
 961                         di->postbuf = NULL;
 962                         di->postbufsize = 0;
 963                 }
 964
 965                 downloads = di;
 966                 return true;
 967         }
 968 }
 969
 970 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap)
 971 {
 972         return Curl_Begin(URL, NULL, maxspeed, name, ispak, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
 973 }
 974 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 975 {
 976         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
 977 }
 978 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 979 {
 980         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
 981 }
 982
 983 /*
 984 ====================
 985 Curl_Run
 986
 987 call this regularily as this will always download as much as possible without
 988 blocking.
 989 ====================
 990 */
 991 void Curl_Run(void)
 992 {
 993         double maxspeed;
 994         downloadinfo *di;
 995
 996         noclear = FALSE;
 997
 998         if(!cl_curl_enabled.integer)
 999                 return;
1000
1001         if(!curl_dll)
1002                 return;
1003
1004         Curl_CheckCommandWhenDone();
1005
1006         if(!downloads)
1007                 return;
1008
1009         if(realtime < curltime) // throttle
1010                 return;
1011
1012         {
1013                 int remaining;
1014                 CURLMcode mc;
1015
1016                 do
1017                 {
1018                         mc = qcurl_multi_perform(curlm, &remaining);
1019                 }
1020                 while(mc == CURLM_CALL_MULTI_PERFORM);
1021
1022                 for(di = downloads; di; di = di->next)
1023                 {
1024                         double b = 0;
1025                         qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1026                         bytes_sent += (b - di->bytes_sent_curl);
1027                         di->bytes_sent_curl = b;
1028                         qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1029                         bytes_sent += (b - di->bytes_received_curl);
1030                         di->bytes_received_curl = b;
1031                 }
1032
1033                 for(;;)
1034                 {
1035                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1036                         if(!msg)
1037                                 break;
1038                         if(msg->msg == CURLMSG_DONE)
1039                         {
1040                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1041                                 CURLcode result;
1042                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1043                                 result = msg->data.result;
1044                                 if(result)
1045                                 {
1046                                         failed = CURL_DOWNLOAD_FAILED;
1047                                 }
1048                                 else
1049                                 {
1050                                         long code;
1051                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1052                                         switch(code / 100)
1053                                         {
1054                                                 case 4: // e.g. 404?
1055                                                 case 5: // e.g. 500?
1056                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1057                                                         result = (CURLcode) code;
1058                                                         break;
1059                                         }
1060                                 }
1061
1062                                 Curl_EndDownload(di, failed, result);
1063                         }
1064                 }
1065         }
1066
1067         CheckPendingDownloads();
1068
1069         // when will we curl the next time?
1070         // we will wait a bit to ensure our download rate is kept.
1071         // we now know that realtime >= curltime... so set up a new curltime
1072
1073         // use the slowest allowing download to derive the maxspeed... this CAN
1074         // be done better, but maybe later
1075         maxspeed = cl_curl_maxspeed.value;
1076         for(di = downloads; di; di = di->next)
1077                 if(di->maxspeed > 0)
1078                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1079                                 maxspeed = di->maxspeed;
1080
1081         if(maxspeed > 0)
1082         {
1083                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1084                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1085                 bytes_sent = 0;
1086                 bytes_received = 0;
1087         }
1088         else
1089                 curltime = realtime;
1090 }
1091
1092 /*
1093 ====================
1094 Curl_CancelAll
1095
1096 Stops ALL downloads.
1097 ====================
1098 */
1099 void Curl_CancelAll(void)
1100 {
1101         if(!curl_dll)
1102                 return;
1103
1104         while(downloads)
1105         {
1106                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1107                 // INVARIANT: downloads will point to the next download after that!
1108         }
1109 }
1110
1111 /*
1112 ====================
1113 Curl_Running
1114
1115 returns true iff there is a download running.
1116 ====================
1117 */
1118 qboolean Curl_Running(void)
1119 {
1120         if(!curl_dll)
1121                 return false;
1122
1123         return downloads != NULL;
1124 }
1125
1126 /*
1127 ====================
1128 Curl_GetDownloadAmount
1129
1130 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1131 for the given download.
1132 ====================
1133 */
1134 static double Curl_GetDownloadAmount(downloadinfo *di)
1135 {
1136         if(!curl_dll)
1137                 return -2;
1138         if(di->curle)
1139         {
1140                 double length;
1141                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1142                 if(length > 0)
1143                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1144                 else
1145                         return 0;
1146         }
1147         else
1148                 return -1;
1149 }
1150
1151 /*
1152 ====================
1153 Curl_GetDownloadSpeed
1154
1155 returns the speed of the given download in bytes per second
1156 ====================
1157 */
1158 static double Curl_GetDownloadSpeed(downloadinfo *di)
1159 {
1160         if(!curl_dll)
1161                 return -2;
1162         if(di->curle)
1163         {
1164                 double speed;
1165                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1166                 return speed;
1167         }
1168         else
1169                 return -1;
1170 }
1171
1172 /*
1173 ====================
1174 Curl_Info_f
1175
1176 prints the download list
1177 ====================
1178 */
1179 // TODO rewrite using Curl_GetDownloadInfo?
1180 static void Curl_Info_f(void)
1181 {
1182         downloadinfo *di;
1183         if(!curl_dll)
1184                 return;
1185         if(Curl_Running())
1186         {
1187                 Con_Print("Currently running downloads:\n");
1188                 for(di = downloads; di; di = di->next)
1189                 {
1190                         double speed, percent;
1191                         Con_Printf("  %s -> %s ",  CleanURL(di->url), di->filename);
1192                         percent = 100.0 * Curl_GetDownloadAmount(di);
1193                         speed = Curl_GetDownloadSpeed(di);
1194                         if(percent >= 0)
1195                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1196                         else
1197                                 Con_Print("(queued)\n");
1198                 }
1199         }
1200         else
1201         {
1202                 Con_Print("No downloads running.\n");
1203         }
1204 }
1205
1206 /*
1207 ====================
1208 Curl_Curl_f
1209
1210 implements the "curl" console command
1211
1212 curl --info
1213 curl --cancel
1214 curl --cancel filename
1215 curl url
1216
1217 For internal use:
1218
1219 curl [--pak] [--forthismap] [--for filename filename...] url
1220         --pak: after downloading, load the package into the virtual file system
1221         --for filename...: only download of at least one of the named files is missing
1222         --forthismap: don't reconnect on failure
1223
1224 curl --clear_autodownload
1225         clears the download success/failure counters
1226
1227 curl --finish_autodownload
1228         if at least one download has been started, disconnect and drop to the menu
1229         once the last download completes successfully, reconnect to the current server
1230 ====================
1231 */
1232 void Curl_Curl_f(void)
1233 {
1234         double maxspeed = 0;
1235         int i;
1236         int end;
1237         qboolean pak = false;
1238         qboolean forthismap = false;
1239         const char *url;
1240         const char *name = 0;
1241
1242         if(!curl_dll)
1243         {
1244                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1245                 return;
1246         }
1247
1248         if(!cl_curl_enabled.integer)
1249         {
1250                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1251                 return;
1252         }
1253
1254         if(Cmd_Argc() < 2)
1255         {
1256                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1257                 return;
1258         }
1259
1260         url = Cmd_Argv(Cmd_Argc() - 1);
1261         end = Cmd_Argc();
1262
1263         for(i = 1; i != end; ++i)
1264         {
1265                 const char *a = Cmd_Argv(i);
1266                 if(!strcmp(a, "--info"))
1267                 {
1268                         Curl_Info_f();
1269                         return;
1270                 }
1271                 else if(!strcmp(a, "--cancel"))
1272                 {
1273                         if(i == end - 1) // last argument
1274                                 Curl_CancelAll();
1275                         else
1276                         {
1277                                 downloadinfo *di = Curl_Find(url);
1278                                 if(di)
1279                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1280                                 else
1281                                         Con_Print("download not found\n");
1282                         }
1283                         return;
1284                 }
1285                 else if(!strcmp(a, "--pak"))
1286                 {
1287                         pak = true;
1288                 }
1289                 else if(!strcmp(a, "--for")) // must be last option
1290                 {
1291                         for(i = i + 1; i != end - 1; ++i)
1292                         {
1293                                 if(!FS_FileExists(Cmd_Argv(i)))
1294                                         goto needthefile; // why can't I have a "double break"?
1295                         }
1296                         // if we get here, we have all the files...
1297                         return;
1298                 }
1299                 else if(!strcmp(a, "--forthismap"))
1300                 {
1301                         forthismap = true;
1302                 }
1303                 else if(!strcmp(a, "--as"))
1304                 {
1305                         if(i < end - 1)
1306                         {
1307                                 ++i;
1308                                 name = Cmd_Argv(i);
1309                         }
1310                 }
1311                 else if(!strcmp(a, "--clear_autodownload"))
1312                 {
1313                         // mark all running downloads as "not for this map", so if they
1314                         // fail, it does not matter
1315                         Curl_Clear_forthismap();
1316                         return;
1317                 }
1318                 else if(!strcmp(a, "--finish_autodownload"))
1319                 {
1320                         if(numdownloads_added)
1321                         {
1322                                 char donecommand[256];
1323                                 if(cls.netcon)
1324                                 {
1325                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1326                                         {
1327                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1328                                                 Curl_CommandWhenDone(donecommand);
1329                                                 noclear = TRUE;
1330                                                 CL_Disconnect();
1331                                                 noclear = FALSE;
1332                                                 Curl_CheckCommandWhenDone();
1333                                         }
1334                                         else
1335                                                 Curl_Register_predownload();
1336                                 }
1337                         }
1338                         return;
1339                 }
1340                 else if(!strncmp(a, "--maxspeed=", 11))
1341                 {
1342                         maxspeed = atof(a + 11);
1343                 }
1344                 else if(*a == '-')
1345                 {
1346                         Con_Printf("curl: invalid option %s\n", a);
1347                         // but we ignore the option
1348                 }
1349         }
1350
1351 needthefile:
1352         Curl_Begin_ToFile(url, maxspeed, name, pak, forthismap);
1353 }
1354
1355 /*
1356 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1357 {
1358         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1359         Z_Free(buffer);
1360 }
1361
1362 void Curl_CurlCat_f(void)
1363 {
1364         unsigned char *buf;
1365         const char *url = Cmd_Argv(1);
1366         buf = Z_Malloc(16384);
1367         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1368 }
1369 */
1370
1371 /*
1372 ====================
1373 Curl_Init_Commands
1374
1375 loads the commands and cvars this library uses
1376 ====================
1377 */
1378 void Curl_Init_Commands(void)
1379 {
1380         Cvar_RegisterVariable (&cl_curl_enabled);
1381         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1382         Cvar_RegisterVariable (&cl_curl_maxspeed);
1383         Cvar_RegisterVariable (&sv_curl_defaulturl);
1384         Cvar_RegisterVariable (&sv_curl_serverpackages);
1385         Cvar_RegisterVariable (&sv_curl_maxspeed);
1386         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1387         //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1388 }
1389
1390 /*
1391 ====================
1392 Curl_GetDownloadInfo
1393
1394 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1395 The number of elements in the array is returned in int *nDownloads.
1396 const char **additional_info may be set to a string of additional user
1397 information, or to NULL if no such display shall occur. The returned
1398 array must be freed later using Z_Free.
1399 ====================
1400 */
1401 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1402 {
1403         int i;
1404         downloadinfo *di;
1405         Curl_downloadinfo_t *downinfo;
1406         static char addinfo[128];
1407
1408         if(!curl_dll)
1409         {
1410                 *nDownloads = 0;
1411                 if(additional_info)
1412                         *additional_info = NULL;
1413                 return NULL;
1414         }
1415
1416         i = 0;
1417         for(di = downloads; di; di = di->next)
1418                 ++i;
1419
1420         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1421         i = 0;
1422         for(di = downloads; di; di = di->next)
1423         {
1424                 // do not show infobars for background downloads
1425                 if(developer.integer <= 0)
1426                         if(di->buffer)
1427                                 continue;
1428                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1429                 if(di->curle)
1430                 {
1431                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1432                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1433                         downinfo[i].queued = false;
1434                 }
1435                 else
1436                 {
1437                         downinfo[i].queued = true;
1438                 }
1439                 ++i;
1440         }
1441
1442         if(additional_info)
1443         {
1444                 // TODO: can I clear command_when_done as soon as the first download fails?
1445                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1446                 {
1447                         if(!strncmp(command_when_done, "connect ", 8))
1448                                 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1449                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1450                                 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1451                         else
1452                                 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1453                         *additional_info = addinfo;
1454                 }
1455                 else
1456                         *additional_info = NULL;
1457         }
1458
1459         *nDownloads = i;
1460         return downinfo;
1461 }
1462
1463
1464 /*
1465 ====================
1466 Curl_FindPackURL
1467
1468 finds the URL where to find a given package.
1469
1470 For this, it reads a file "curl_urls.txt" of the following format:
1471
1472         data*.pk3       -
1473         revdm*.pk3      http://revdm/downloads/are/here/
1474         *                       http://any/other/stuff/is/here/
1475
1476 The URLs should end in /. If not, downloads will still work, but the cached files
1477 can't be just put into the data directory with the same download configuration
1478 (you might want to do this if you want to tag downloaded files from your
1479 server, but you should not). "-" means "don't download".
1480
1481 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1482 location instead.
1483
1484 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1485 this file for obvious reasons.
1486 ====================
1487 */
1488 static const char *Curl_FindPackURL(const char *filename)
1489 {
1490         static char foundurl[1024];
1491         fs_offset_t filesize;
1492         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1493         if(buf && filesize)
1494         {
1495                 // read lines of format "pattern url"
1496                 char *p = buf;
1497                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1498                 qboolean eof = false;
1499
1500                 pattern = p;
1501                 while(!eof)
1502                 {
1503                         switch(*p)
1504                         {
1505                                 case 0:
1506                                         eof = true;
1507                                         // fallthrough
1508                                 case '\n':
1509                                 case '\r':
1510                                         if(pattern && url && patternend)
1511                                         {
1512                                                 if(!urlend)
1513                                                         urlend = p;
1514                                                 *patternend = 0;
1515                                                 *urlend = 0;
1516                                                 if(matchpattern(filename, pattern, true))
1517                                                 {
1518                                                         strlcpy(foundurl, url, sizeof(foundurl));
1519                                                         Z_Free(buf);
1520                                                         return foundurl;
1521                                                 }
1522                                         }
1523                                         pattern = NULL;
1524                                         patternend = NULL;
1525                                         url = NULL;
1526                                         urlend = NULL;
1527                                         break;
1528                                 case ' ':
1529                                 case '\t':
1530                                         if(pattern && !patternend)
1531                                                 patternend = p;
1532                                         else if(url && !urlend)
1533                                                 urlend = p;
1534                                         break;
1535                                 default:
1536                                         if(!pattern)
1537                                                 pattern = p;
1538                                         else if(pattern && patternend && !url)
1539                                                 url = p;
1540                                         break;
1541                         }
1542                         ++p;
1543                 }
1544         }
1545         if(buf)
1546                 Z_Free(buf);
1547         return sv_curl_defaulturl.string;
1548 }
1549
1550 typedef struct requirement_s
1551 {
1552         struct requirement_s *next;
1553         char filename[MAX_OSPATH];
1554 }
1555 requirement;
1556 static requirement *requirements = NULL;
1557
1558
1559 /*
1560 ====================
1561 Curl_RequireFile
1562
1563 Adds the given file to the list of requirements.
1564 ====================
1565 */
1566 void Curl_RequireFile(const char *filename)
1567 {
1568         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1569         req->next = requirements;
1570         strlcpy(req->filename, filename, sizeof(req->filename));
1571         requirements = req;
1572 }
1573
1574 /*
1575 ====================
1576 Curl_ClearRequirements
1577
1578 Clears the list of required files for playing on the current map.
1579 This should be called at every map change.
1580 ====================
1581 */
1582 void Curl_ClearRequirements(void)
1583 {
1584         while(requirements)
1585         {
1586                 requirement *req = requirements;
1587                 requirements = requirements->next;
1588                 Z_Free(req);
1589         }
1590 }
1591
1592 /*
1593 ====================
1594 Curl_SendRequirements
1595
1596 Makes the current host_clients download all files he needs.
1597 This is done by sending him the following console commands:
1598
1599         curl --clear_autodownload
1600         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1601         curl --finish_autodownload
1602 ====================
1603 */
1604 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1605 {
1606         const char *p;
1607         const char *thispack = FS_WhichPack(filename);
1608         const char *packurl;
1609
1610         if(!thispack)
1611                 return false;
1612
1613         p = strrchr(thispack, '/');
1614         if(p)
1615                 thispack = p + 1;
1616
1617         packurl = Curl_FindPackURL(thispack);
1618
1619         if(packurl && *packurl && strcmp(packurl, "-"))
1620         {
1621                 if(!foundone)
1622                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1623
1624                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1625                 strlcat(sendbuffer, thispack, sendbuffer_len);
1626                 if(sv_curl_maxspeed.value > 0)
1627                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1628                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1629                 strlcat(sendbuffer, filename, sendbuffer_len);
1630                 strlcat(sendbuffer, " ", sendbuffer_len);
1631                 strlcat(sendbuffer, packurl, sendbuffer_len);
1632                 strlcat(sendbuffer, thispack, sendbuffer_len);
1633                 strlcat(sendbuffer, "\n", sendbuffer_len);
1634
1635                 return true;
1636         }
1637
1638         return false;
1639 }
1640 void Curl_SendRequirements(void)
1641 {
1642         // for each requirement, find the pack name
1643         char sendbuffer[4096] = "";
1644         requirement *req;
1645         qboolean foundone = false;
1646         const char *p;
1647
1648         for(req = requirements; req; req = req->next)
1649                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1650
1651         p = sv_curl_serverpackages.string;
1652         while(COM_ParseToken_Simple(&p, false, false))
1653                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1654
1655         if(foundone)
1656                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1657
1658         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1659                 Host_ClientCommands("%s", sendbuffer);
1660         else
1661                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1662 }