libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {1, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {1, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {1, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t sv_curl_serverpackages = {1, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
   9 static cvar_t cl_curl_enabled = {1, "cl_curl_enabled","0", "whether client's download support is enabled"};
  10
  11 /*
  12 =================================================================
  13
  14   Minimal set of definitions from libcurl
  15
  16   WARNING: for a matter of simplicity, several pointer types are
  17   casted to "void*", and most enumerated values are not included
  18
  19 =================================================================
  20 */
  21
  22 typedef struct CURL_s CURL;
  23 typedef struct CURLM_s CURLM;
  24 typedef enum
  25 {
  26         CURLE_OK = 0
  27 }
  28 CURLcode;
  29 typedef enum
  30 {
  31         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  32         CURLM_OK = 0
  33 }
  34 CURLMcode;
  35 #define CURL_GLOBAL_NOTHING 0
  36 #define CURL_GLOBAL_SSL 1
  37 #define CURL_GLOBAL_WIN32 2
  38 #define CURLOPTTYPE_LONG          0
  39 #define CURLOPTTYPE_OBJECTPOINT   10000
  40 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  41 #define CURLOPTTYPE_OFF_T         30000
  42 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  43 typedef enum
  44 {
  45         CINIT(WRITEDATA, OBJECTPOINT, 1),
  46         CINIT(URL,  OBJECTPOINT, 2),
  47         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  48         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  49         CINIT(REFERER, OBJECTPOINT, 16),
  50         CINIT(USERAGENT, OBJECTPOINT, 18),
  51         CINIT(RESUME_FROM, LONG, 21),
  52         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  53         CINIT(PRIVATE, OBJECTPOINT, 103),
  54         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  55         CINIT(LOW_SPEED_TIME, LONG, 20),
  56 }
  57 CURLoption;
  58 typedef enum
  59 {
  60         CURLINFO_TEXT = 0,
  61         CURLINFO_HEADER_IN,    /* 1 */
  62         CURLINFO_HEADER_OUT,   /* 2 */
  63         CURLINFO_DATA_IN,      /* 3 */
  64         CURLINFO_DATA_OUT,     /* 4 */
  65         CURLINFO_SSL_DATA_IN,  /* 5 */
  66         CURLINFO_SSL_DATA_OUT, /* 6 */
  67         CURLINFO_END
  68 }
  69 curl_infotype;
  70 #define CURLINFO_STRING   0x100000
  71 #define CURLINFO_LONG     0x200000
  72 #define CURLINFO_DOUBLE   0x300000
  73 #define CURLINFO_SLIST    0x400000
  74 #define CURLINFO_MASK     0x0fffff
  75 #define CURLINFO_TYPEMASK 0xf00000
  76 typedef enum
  77 {
  78         CURLINFO_NONE, /* first, never use this */
  79         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  80         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  81         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  82         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  83         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  84         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  85         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  86         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  87         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  88         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
  89         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
  90         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
  91         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
  92         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
  93         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
  94         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
  95         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
  96         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
  97         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
  98         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
  99         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 100         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 101         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 102         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 103         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 104         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 105         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
 106 }
 107 CURLINFO;
 108
 109 typedef enum
 110 {
 111         CURLMSG_NONE, /* first, not used */
 112         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 113                                          the CURLcode of the transfer */
 114         CURLMSG_LAST
 115 }
 116 CURLMSG;
 117 typedef struct
 118 {
 119         CURLMSG msg;       /* what this message means */
 120         CURL *easy_handle; /* the handle it concerns */
 121         union
 122         {
 123                 void *whatever;    /* message-specific data */
 124                 CURLcode result;   /* return code for transfer */
 125         }
 126         data;
 127 }
 128 CURLMsg;
 129
 130 static void (*qcurl_global_init) (long flags);
 131 static void (*qcurl_global_cleanup) ();
 132
 133 static CURL * (*qcurl_easy_init) ();
 134 static void (*qcurl_easy_cleanup) (CURL *handle);
 135 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 136 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 137 static const char * (*qcurl_easy_strerror) (CURLcode);
 138
 139 static CURLM * (*qcurl_multi_init) ();
 140 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 141 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 142 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 143 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 144 static void (*qcurl_multi_cleanup) (CURLM *);
 145 static const char * (*qcurl_multi_strerror) (CURLcode);
 146
 147 static dllfunction_t curlfuncs[] =
 148 {
 149         {"curl_global_init",            (void **) &qcurl_global_init},
 150         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 151         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 152         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 153         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 154         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 155         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 156         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 157         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 158         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 159         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 160         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 161         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 162         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 163         {NULL, NULL}
 164 };
 165
 166 // Handle for CURL DLL
 167 static dllhandle_t curl_dll = NULL;
 168 // will be checked at many places to find out if qcurl calls are allowed
 169
 170 typedef struct downloadinfo_s
 171 {
 172         char filename[MAX_QPATH];
 173         char url[256];
 174         char referer[256];
 175         qfile_t *stream;
 176         fs_offset_t startpos;
 177         CURL *curle;
 178         qboolean started;
 179         qboolean ispak;
 180         unsigned long bytes_received;
 181         struct downloadinfo_s *next, *prev;
 182         qboolean forthismap;
 183 }
 184 downloadinfo;
 185 static downloadinfo *downloads = NULL;
 186 static int numdownloads = 0;
 187
 188 static int numdownloads_fail = 0;
 189 static int numdownloads_success = 0;
 190 static int numdownloads_added = 0;
 191 static char command_when_done[256] = "";
 192 static char command_when_error[256] = "";
 193
 194 /*
 195 ====================
 196 Curl_CommandWhenDone
 197
 198 Sets the command which is to be executed when the last download completes AND
 199 all downloads since last server connect ended with a successful status.
 200 Setting the command to NULL clears it.
 201 ====================
 202 */
 203 void Curl_CommandWhenDone(const char *cmd)
 204 {
 205         if(!curl_dll)
 206                 return;
 207         if(cmd)
 208                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 209         else
 210                 *command_when_done = 0;
 211 }
 212
 213 /*
 214 FIXME
 215 Do not use yet. Not complete.
 216 Problem: what counts as an error?
 217 */
 218
 219 void Curl_CommandWhenError(const char *cmd)
 220 {
 221         if(!curl_dll)
 222                 return;
 223         if(cmd)
 224                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 225         else
 226                 *command_when_error = 0;
 227 }
 228
 229 /*
 230 ====================
 231 Curl_Clear_forthismap
 232
 233 Clears the "will disconnect on failure" flags.
 234 ====================
 235 */
 236 void Curl_Clear_forthismap()
 237 {
 238         downloadinfo *di;
 239         for(di = downloads; di; di = di->next)
 240                 di->forthismap = false;
 241         Curl_CommandWhenError(NULL);
 242         Curl_CommandWhenDone(NULL);
 243         numdownloads_fail = 0;
 244         numdownloads_success = 0;
 245         numdownloads_added = 0;
 246 }
 247
 248 /* obsolete: numdownloads_added contains the same
 249 static qboolean Curl_Have_forthismap()
 250 {
 251         downloadinfo *di;
 252         for(di = downloads; di; di = di->next)
 253                 if(di->forthismap)
 254                         return true;
 255         return false;
 256 }
 257 */
 258
 259 /*
 260 ====================
 261 Curl_CheckCommandWhenDone
 262
 263 Checks if a "done command" is to be executed.
 264 All downloads finished, at least one success since connect, no single failure
 265 -> execute the command.
 266 */
 267 static void Curl_CheckCommandWhenDone()
 268 {
 269         if(!curl_dll)
 270                 return;
 271         if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
 272         {
 273                 Con_DPrintf("Map downloads occurred, executing %s\n", command_when_done);
 274                 Cbuf_AddText("\n");
 275                 Cbuf_AddText(command_when_done);
 276                 Cbuf_AddText("\n");
 277                 Curl_Clear_forthismap();
 278         }
 279         else if(numdownloads_added && numdownloads_fail && *command_when_error)
 280         {
 281                 Con_DPrintf("Map downloads FAILED, executing %s\n", command_when_error);
 282                 Cbuf_AddText("\n");
 283                 Cbuf_AddText(command_when_error);
 284                 Cbuf_AddText("\n");
 285                 Curl_Clear_forthismap();
 286         }
 287 }
 288
 289 /*
 290 ====================
 291 CURL_CloseLibrary
 292
 293 Load the cURL DLL
 294 ====================
 295 */
 296 static qboolean CURL_OpenLibrary (void)
 297 {
 298         const char* dllnames [] =
 299         {
 300 #if defined(WIN64)
 301                 "libcurl64.dll",
 302 #elif defined(WIN32)
 303                 "libcurl-3.dll",
 304 #elif defined(MACOSX)
 305                 "libcurl.3.dylib", // Mac OS X Tiger
 306                 "libcurl.2.dylib", // Mac OS X Panther
 307 #else
 308                 "libcurl.so.3",
 309 #endif
 310                 NULL
 311         };
 312
 313         // Already loaded?
 314         if (curl_dll)
 315                 return true;
 316
 317         // Load the DLL
 318         if (! Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs))
 319         {
 320                 Con_Printf ("cURL support disabled\n");
 321                 return false;
 322         }
 323
 324         Con_Printf ("cURL support enabled\n");
 325         return true;
 326 }
 327
 328
 329 /*
 330 ====================
 331 CURL_CloseLibrary
 332
 333 Unload the cURL DLL
 334 ====================
 335 */
 336 static void CURL_CloseLibrary (void)
 337 {
 338         Sys_UnloadLibrary (&curl_dll);
 339 }
 340
 341
 342 static CURLM *curlm = NULL;
 343 static unsigned long bytes_received = 0; // used for bandwidth throttling
 344 static double curltime = 0;
 345
 346 /*
 347 ====================
 348 CURL_fwrite
 349
 350 fwrite-compatible function that writes the data to a file. libcurl can call
 351 this.
 352 ====================
 353 */
 354 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 355 {
 356         fs_offset_t ret;
 357         size_t bytes = size * nmemb;
 358         downloadinfo *di = (downloadinfo *) vdi;
 359
 360         bytes_received += bytes;
 361         di->bytes_received += bytes;
 362
 363         ret = FS_Write(di->stream, data, bytes);
 364
 365         return ret; // why not ret / nmemb?
 366 }
 367
 368 typedef enum
 369 {
 370         CURL_DOWNLOAD_SUCCESS = 0,
 371         CURL_DOWNLOAD_FAILED,
 372         CURL_DOWNLOAD_ABORTED,
 373         CURL_DOWNLOAD_SERVERERROR
 374 }
 375 CurlStatus;
 376
 377 /*
 378 ====================
 379 Curl_EndDownload
 380
 381 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 382 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 383 code from libcurl, or 0, if another error has occurred.
 384 ====================
 385 */
 386 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 387 {
 388         qboolean ok = false;
 389         if(!curl_dll)
 390                 return;
 391         switch(status)
 392         {
 393                 case CURL_DOWNLOAD_SUCCESS:
 394                         Con_Printf("Download of %s: OK\n", di->filename);
 395                         ok = true;
 396                         break;
 397                 case CURL_DOWNLOAD_FAILED:
 398                         Con_Printf("Download of %s: FAILED\n", di->filename);
 399                         if(error)
 400                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
 401                         break;
 402                 case CURL_DOWNLOAD_ABORTED:
 403                         Con_Printf("Download of %s: ABORTED\n", di->filename);
 404                         break;
 405                 case CURL_DOWNLOAD_SERVERERROR:
 406                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
 407
 408                         // reopen to enforce it to have zero bytes again
 409                         FS_Close(di->stream);
 410                         di->stream = FS_Open(di->filename, "w", false, false);
 411
 412                         break;
 413         }
 414
 415         if(di->curle)
 416         {
 417                 qcurl_multi_remove_handle(curlm, di->curle);
 418                 qcurl_easy_cleanup(di->curle);
 419         }
 420
 421         if(ok && !di->bytes_received)
 422         {
 423                 Con_Printf("ERROR: empty file\n");
 424                 ok = false;
 425         }
 426
 427         if(di->stream)
 428                 FS_Close(di->stream);
 429
 430         if(ok && di->ispak)
 431                 ok = FS_AddPack(di->filename, NULL, true);
 432
 433         if(di->prev)
 434                 di->prev->next = di->next;
 435         else
 436                 downloads = di->next;
 437         if(di->next)
 438                 di->next->prev = di->prev;
 439
 440         --numdownloads;
 441         if(di->forthismap)
 442         {
 443                 if(ok)
 444                         ++numdownloads_success;
 445                 else
 446                         ++numdownloads_fail;
 447         }
 448         Z_Free(di);
 449
 450         Curl_CheckCommandWhenDone();
 451 }
 452
 453 /*
 454 ====================
 455 CheckPendingDownloads
 456
 457 checks if there are free download slots to start new downloads in.
 458 To not start too many downloads at once, only one download is added at a time,
 459 up to a maximum number of cl_curl_maxdownloads are running.
 460 ====================
 461 */
 462 static void CheckPendingDownloads()
 463 {
 464         if(!curl_dll)
 465                 return;
 466         if(numdownloads < cl_curl_maxdownloads.integer)
 467         {
 468                 downloadinfo *di;
 469                 for(di = downloads; di; di = di->next)
 470                 {
 471                         if(!di->started)
 472                         {
 473                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
 474
 475                                 di->stream = FS_Open(di->filename, "ab", false, false);
 476                                 if(!di->stream)
 477                                 {
 478                                         Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 479                                         Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 480                                         return;
 481                                 }
 482
 483                                 FS_Seek(di->stream, 0, SEEK_END);
 484                                 di->startpos = FS_Tell(di->stream);
 485                                 if(di->startpos > 0)
 486                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
 487                                 Con_Print("...\n");
 488
 489                                 di->curle = qcurl_easy_init();
 490                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 491                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 492                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 493                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 494                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 495                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 496                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 497                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 498                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 499                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 500                                 qcurl_multi_add_handle(curlm, di->curle);
 501                                 di->started = true;
 502                                 ++numdownloads;
 503                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 504                                         break;
 505                         }
 506                 }
 507         }
 508 }
 509
 510 /*
 511 ====================
 512 Curl_Init
 513
 514 this function MUST be called before using anything else in this file.
 515 On Win32, this must be called AFTER WSAStartup has been done!
 516 ====================
 517 */
 518 void Curl_Init()
 519 {
 520         CURL_OpenLibrary();
 521         if(!curl_dll)
 522                 return;
 523         qcurl_global_init(CURL_GLOBAL_NOTHING);
 524         curlm = qcurl_multi_init();
 525 }
 526
 527 /*
 528 ====================
 529 Curl_Shutdown
 530
 531 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 532 ====================
 533 */
 534 void Curl_ClearRequirements();
 535 void Curl_Shutdown()
 536 {
 537         if(!curl_dll)
 538                 return;
 539         Curl_ClearRequirements();
 540         Curl_CancelAll();
 541         CURL_CloseLibrary();
 542         curl_dll = NULL;
 543 }
 544
 545 /*
 546 ====================
 547 Curl_Find
 548
 549 Finds the internal information block for a download given by file name.
 550 ====================
 551 */
 552 static downloadinfo *Curl_Find(const char *filename)
 553 {
 554         downloadinfo *di;
 555         if(!curl_dll)
 556                 return NULL;
 557         for(di = downloads; di; di = di->next)
 558                 if(!strcasecmp(di->filename, filename))
 559                         return di;
 560         return NULL;
 561 }
 562
 563 /*
 564 ====================
 565 Curl_Begin
 566
 567 Starts a download of a given URL to the file name portion of this URL (or name
 568 if given) in the "dlcache/" folder.
 569 ====================
 570 */
 571 void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
 572 {
 573         if(!curl_dll)
 574                 return;
 575         else
 576         {
 577                 char fn[MAX_QPATH];
 578                 const char *p, *q;
 579                 size_t length;
 580                 downloadinfo *di;
 581
 582                 // Note: This extraction of the file name portion is NOT entirely correct.
 583                 //
 584                 // It does the following:
 585                 //
 586                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 587                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 588                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 589                 //
 590                 // However, I'd like to keep this "buggy" behavior so that PHP script
 591                 // authors can write download scripts without having to enable
 592                 // AcceptPathInfo on Apache. They just have to ensure that their script
 593                 // can be called with such a "fake" path name like
 594                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 595                 //
 596                 // By the way, such PHP scripts should either send the file or a
 597                 // "Location:" redirect; PHP code example:
 598                 //
 599                 //   header("Location: http://www.example.com/");
 600                 //
 601                 // By the way, this will set User-Agent to something like
 602                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 603                 // dp://serverhost:serverport/ so you can filter on this; an example
 604                 // httpd log file line might be:
 605                 //
 606                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 607
 608                 if(!name)
 609                         name = URL;
 610                 p = strrchr(name, '/');
 611                 p = p ? (p+1) : name;
 612                 q = strchr(p, '?');
 613                 length = q ? (size_t)(q - p) : strlen(p);
 614                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 615
 616
 617                 // already downloading the file?
 618                 {
 619                         downloadinfo *di = Curl_Find(fn);
 620                         if(di)
 621                         {
 622                                 Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
 623
 624                                 // however, if it was not for this map yet...
 625                                 if(forthismap && !di->forthismap)
 626                                 {
 627                                         di->forthismap = true;
 628                                         // this "fakes" a download attempt so the client will wait for
 629                                         // the download to finish and then reconnect
 630                                         ++numdownloads_added;
 631                                 }
 632
 633                                 return;
 634                         }
 635                 }
 636
 637                 if(ispak && FS_FileExists(fn))
 638                 {
 639                         qboolean already_loaded;
 640                         if(FS_AddPack(fn, &already_loaded, true))
 641                         {
 642                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 643                                 if(already_loaded)
 644                                         Con_DPrintf("(pak was already loaded)\n");
 645                                 else
 646                                 {
 647                                         if(forthismap)
 648                                         {
 649                                                 ++numdownloads_added;
 650                                                 ++numdownloads_success;
 651                                         }
 652                                 }
 653                                 return;
 654                         }
 655                         else
 656                         {
 657                                 qfile_t *f = FS_Open(fn, "rb", false, false);
 658                                 if(f)
 659                                 {
 660                                         char buf[4] = {0};
 661                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 662
 663                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 664                                         {
 665                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 666                                                 FS_Close(f);
 667                                                 f = FS_Open(fn, "w", false, false);
 668                                                 if(f)
 669                                                         FS_Close(f);
 670                                         }
 671                                         else
 672                                         {
 673                                                 // OK
 674                                                 FS_Close(f);
 675                                         }
 676                                 }
 677                         }
 678                 }
 679
 680                 if(forthismap)
 681                         ++numdownloads_added;
 682                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 683                 strlcpy(di->filename, fn, sizeof(di->filename));
 684                 strlcpy(di->url, URL, sizeof(di->url));
 685                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 686                 di->forthismap = forthismap;
 687                 di->stream = NULL;
 688                 di->startpos = 0;
 689                 di->curle = NULL;
 690                 di->started = false;
 691                 di->ispak = ispak;
 692                 di->bytes_received = 0;
 693                 di->next = downloads;
 694                 di->prev = NULL;
 695                 if(di->next)
 696                         di->next->prev = di;
 697                 downloads = di;
 698         }
 699 }
 700
 701
 702 /*
 703 ====================
 704 Curl_Run
 705
 706 call this regularily as this will always download as much as possible without
 707 blocking.
 708 ====================
 709 */
 710 void Curl_Run()
 711 {
 712         if(!cl_curl_enabled.integer)
 713                 return;
 714
 715         if(!curl_dll)
 716                 return;
 717
 718         if(!downloads)
 719                 return;
 720
 721         if(realtime < curltime) // throttle
 722                 return;
 723
 724         {
 725                 int remaining;
 726                 CURLMcode mc;
 727
 728                 do
 729                 {
 730                         mc = qcurl_multi_perform(curlm, &remaining);
 731                 }
 732                 while(mc == CURLM_CALL_MULTI_PERFORM);
 733
 734                 for(;;)
 735                 {
 736                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
 737                         if(!msg)
 738                                 break;
 739                         if(msg->msg == CURLMSG_DONE)
 740                         {
 741                                 downloadinfo *di;
 742                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
 743                                 CURLcode result;
 744                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
 745                                 result = msg->data.result;
 746                                 if(result)
 747                                 {
 748                                         failed = CURL_DOWNLOAD_FAILED;
 749                                 }
 750                                 else
 751                                 {
 752                                         long code;
 753                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
 754                                         switch(code / 100)
 755                                         {
 756                                                 case 4: // e.g. 404?
 757                                                 case 5: // e.g. 500?
 758                                                         failed = CURL_DOWNLOAD_SERVERERROR;
 759                                                         result = code;
 760                                                         break;
 761                                         }
 762                                 }
 763
 764                                 Curl_EndDownload(di, failed, result);
 765                         }
 766                 }
 767         }
 768
 769         CheckPendingDownloads();
 770
 771         // when will we curl the next time?
 772         // we will wait a bit to ensure our download rate is kept.
 773         // we now know that realtime >= curltime... so set up a new curltime
 774         if(cl_curl_maxspeed.value > 0)
 775         {
 776                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
 777                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
 778                 bytes_received -= bytes;
 779         }
 780         else
 781                 curltime = realtime;
 782 }
 783
 784 /*
 785 ====================
 786 Curl_CancelAll
 787
 788 Stops ALL downloads.
 789 ====================
 790 */
 791 void Curl_CancelAll()
 792 {
 793         if(!curl_dll)
 794                 return;
 795
 796         while(downloads)
 797         {
 798                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 799                 // INVARIANT: downloads will point to the next download after that!
 800         }
 801 }
 802
 803 /*
 804 ====================
 805 Curl_Running
 806
 807 returns true iff there is a download running.
 808 ====================
 809 */
 810 qboolean Curl_Running()
 811 {
 812         if(!curl_dll)
 813                 return false;
 814
 815         return downloads != NULL;
 816 }
 817
 818 /*
 819 ====================
 820 Curl_GetDownloadAmount
 821
 822 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
 823 for the given download.
 824 ====================
 825 */
 826 static double Curl_GetDownloadAmount(downloadinfo *di)
 827 {
 828         if(!curl_dll)
 829                 return -2;
 830         if(di->curle)
 831         {
 832                 double length;
 833                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
 834                 if(length > 0)
 835                         return di->bytes_received / length;
 836                 else
 837                         return 0;
 838         }
 839         else
 840                 return -1;
 841 }
 842
 843 /*
 844 ====================
 845 Curl_GetDownloadSpeed
 846
 847 returns the speed of the given download in bytes per second
 848 ====================
 849 */
 850 static double Curl_GetDownloadSpeed(downloadinfo *di)
 851 {
 852         if(!curl_dll)
 853                 return -2;
 854         if(di->curle)
 855         {
 856                 double speed;
 857                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
 858                 return speed;
 859         }
 860         else
 861                 return -1;
 862 }
 863
 864 /*
 865 ====================
 866 Curl_Info_f
 867
 868 prints the download list
 869 ====================
 870 */
 871 // TODO rewrite using Curl_GetDownloadInfo?
 872 static void Curl_Info_f()
 873 {
 874         downloadinfo *di;
 875         if(!curl_dll)
 876                 return;
 877         if(Curl_Running())
 878         {
 879                 Con_Print("Currently running downloads:\n");
 880                 for(di = downloads; di; di = di->next)
 881                 {
 882                         double speed, percent;
 883                         Con_Printf("  %s -> %s ",  di->url, di->filename);
 884                         percent = 100.0 * Curl_GetDownloadAmount(di);
 885                         speed = Curl_GetDownloadSpeed(di);
 886                         if(percent >= 0)
 887                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
 888                         else
 889                                 Con_Print("(queued)\n");
 890                 }
 891         }
 892         else
 893         {
 894                 Con_Print("No downloads running.\n");
 895         }
 896 }
 897
 898 /*
 899 ====================
 900 Curl_Curl_f
 901
 902 implements the "curl" console command
 903
 904 curl --info
 905 curl --cancel
 906 curl --cancel filename
 907 curl url
 908
 909 For internal use:
 910
 911 curl [--pak] [--forthismap] [--for filename filename...] url
 912         --pak: after downloading, load the package into the virtual file system
 913         --for filename...: only download of at least one of the named files is missing
 914         --forthismap: don't reconnect on failure
 915
 916 curl --clear_autodownload
 917         clears the download success/failure counters
 918
 919 curl --finish_autodownload
 920         if at least one download has been started, disconnect and drop to the menu
 921         once the last download completes successfully, reconnect to the current server
 922 ====================
 923 */
 924 void Curl_Curl_f(void)
 925 {
 926         int i;
 927         int end;
 928         qboolean pak = false;
 929         qboolean forthismap = false;
 930         const char *url;
 931         const char *name = 0;
 932
 933         if(!curl_dll)
 934         {
 935                 Con_Print("libcurl DLL not found, this command is inactive.\n");
 936                 return;
 937         }
 938
 939         if(!cl_curl_enabled.integer)
 940         {
 941                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
 942                 return;
 943         }
 944
 945         for(i = 0; i != Cmd_Argc(); ++i)
 946                 Con_DPrintf("%s ", Cmd_Argv(i));
 947         Con_DPrint("\n");
 948
 949         if(Cmd_Argc() < 2)
 950         {
 951                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
 952                 return;
 953         }
 954
 955         url = Cmd_Argv(Cmd_Argc() - 1);
 956         end = Cmd_Argc();
 957
 958         for(i = 1; i != end; ++i)
 959         {
 960                 const char *a = Cmd_Argv(i);
 961                 if(!strcmp(a, "--info"))
 962                 {
 963                         Curl_Info_f();
 964                         return;
 965                 }
 966                 else if(!strcmp(a, "--cancel"))
 967                 {
 968                         if(i == end - 1) // last argument
 969                                 Curl_CancelAll();
 970                         else
 971                         {
 972                                 downloadinfo *di = Curl_Find(url);
 973                                 if(di)
 974                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 975                                 else
 976                                         Con_Print("download not found\n");
 977                         }
 978                         return;
 979                 }
 980                 else if(!strcmp(a, "--pak"))
 981                 {
 982                         pak = true;
 983                 }
 984                 else if(!strcmp(a, "--for"))
 985                 {
 986                         for(i = i + 1; i != end - 1; ++i)
 987                         {
 988                                 if(!FS_FileExists(Cmd_Argv(i)))
 989                                         goto needthefile; // why can't I have a "double break"?
 990                         }
 991                         // if we get here, we have all the files...
 992                         return;
 993                 }
 994                 else if(!strcmp(a, "--forthismap"))
 995                 {
 996                         forthismap = true;
 997                 }
 998                 else if(!strcmp(a, "--as"))
 999                 {
1000                         if(i < end - 1)
1001                         {
1002                                 ++i;
1003                                 name = Cmd_Argv(i);
1004                         }
1005                 }
1006                 else if(!strcmp(a, "--clear_autodownload"))
1007                 {
1008                         // mark all running downloads as "not for this map", so if they
1009                         // fail, it does not matter
1010                         Curl_Clear_forthismap();
1011                         return;
1012                 }
1013                 else if(!strcmp(a, "--finish_autodownload"))
1014                 {
1015                         if(numdownloads_added)
1016                         {
1017                                 char donecommand[256];
1018                                 if(cls.netcon)
1019                                 {
1020                                         dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1021                                         Curl_CommandWhenDone(donecommand);
1022                                 }
1023                                 CL_Disconnect();
1024
1025                                 Curl_CheckCommandWhenDone();
1026                         }
1027                         return;
1028                 }
1029                 else if(*a == '-')
1030                 {
1031                         Con_Printf("invalid option %s\n", a);
1032                         return;
1033                 }
1034         }
1035
1036 needthefile:
1037         Curl_Begin(url, name, pak, forthismap);
1038 }
1039
1040 /*
1041 ====================
1042 Curl_Init_Commands
1043
1044 loads the commands and cvars this library uses
1045 ====================
1046 */
1047 void Curl_Init_Commands(void)
1048 {
1049         Cvar_RegisterVariable (&cl_curl_enabled);
1050         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1051         Cvar_RegisterVariable (&cl_curl_maxspeed);
1052         Cvar_RegisterVariable (&sv_curl_defaulturl);
1053         Cvar_RegisterVariable (&sv_curl_serverpackages);
1054         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1055 }
1056
1057 /*
1058 ====================
1059 Curl_GetDownloadInfo
1060
1061 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1062 The number of elements in the array is returned in int *nDownloads.
1063 const char **additional_info may be set to a string of additional user
1064 information, or to NULL if no such display shall occur. The returned
1065 array must be freed later using Z_Free.
1066 ====================
1067 */
1068 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1069 {
1070         int n, i;
1071         downloadinfo *di;
1072         Curl_downloadinfo_t *downinfo;
1073         static char addinfo[128];
1074
1075         if(!curl_dll)
1076         {
1077                 *nDownloads = 0;
1078                 if(additional_info)
1079                         *additional_info = NULL;
1080                 return NULL;
1081         }
1082
1083         n = 0;
1084         for(di = downloads; di; di = di->next)
1085                 ++n;
1086
1087         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
1088         i = 0;
1089         for(di = downloads; di; di = di->next)
1090         {
1091                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1092                 if(di->curle)
1093                 {
1094                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1095                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1096                         downinfo[i].queued = false;
1097                 }
1098                 else
1099                 {
1100                         downinfo[i].queued = true;
1101                 }
1102                 ++i;
1103         }
1104
1105         if(additional_info)
1106         {
1107                 // TODO: can I clear command_when_done as soon as the first download fails?
1108                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1109                 {
1110                         if(strncmp(command_when_done, "connect ", 8))
1111                                 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1112                         else
1113                                 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1114                         *additional_info = addinfo;
1115                 }
1116                 else
1117                         *additional_info = NULL;
1118         }
1119
1120         *nDownloads = n;
1121         return downinfo;
1122 }
1123
1124
1125 /*
1126 ====================
1127 Curl_FindPackURL
1128
1129 finds the URL where to find a given package.
1130
1131 For this, it reads a file "curl_urls.txt" of the following format:
1132
1133         data*.pk3       -
1134         revdm*.pk3      http://revdm/downloads/are/here/
1135         *                       http://any/other/stuff/is/here/
1136
1137 The URLs should end in /. If not, downloads will still work, but the cached files
1138 can't be just put into the data directory with the same download configuration
1139 (you might want to do this if you want to tag downloaded files from your
1140 server, but you should not). "-" means "don't download".
1141
1142 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1143 location instead.
1144
1145 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1146 this file for obvious reasons.
1147 ====================
1148 */
1149 static const char *Curl_FindPackURL(const char *filename)
1150 {
1151         static char foundurl[256];
1152         fs_offset_t filesize;
1153         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1154         if(buf && filesize)
1155         {
1156                 // read lines of format "pattern url"
1157                 char *p = buf;
1158                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1159                 qboolean eof = false;
1160
1161                 pattern = p;
1162                 while(!eof)
1163                 {
1164                         switch(*p)
1165                         {
1166                                 case 0:
1167                                         eof = true;
1168                                         // fallthrough
1169                                 case '\n':
1170                                 case '\r':
1171                                         if(pattern && url && patternend)
1172                                         {
1173                                                 if(!urlend)
1174                                                         urlend = p;
1175                                                 *patternend = 0;
1176                                                 *urlend = 0;
1177                                                 if(matchpattern(filename, pattern, true))
1178                                                 {
1179                                                         strlcpy(foundurl, url, sizeof(foundurl));
1180                                                         Z_Free(buf);
1181                                                         return foundurl;
1182                                                 }
1183                                         }
1184                                         pattern = NULL;
1185                                         patternend = NULL;
1186                                         url = NULL;
1187                                         urlend = NULL;
1188                                         break;
1189                                 case ' ':
1190                                 case '\t':
1191                                         if(pattern && !patternend)
1192                                                 patternend = p;
1193                                         else if(url && !urlend)
1194                                                 urlend = p;
1195                                         break;
1196                                 default:
1197                                         if(!pattern)
1198                                                 pattern = p;
1199                                         else if(pattern && patternend && !url)
1200                                                 url = p;
1201                                         break;
1202                         }
1203                         ++p;
1204                 }
1205         }
1206         if(buf)
1207                 Z_Free(buf);
1208         return sv_curl_defaulturl.string;
1209 }
1210
1211 typedef struct requirement_s
1212 {
1213         struct requirement_s *next;
1214         char filename[MAX_QPATH];
1215 }
1216 requirement;
1217 static requirement *requirements = NULL;
1218
1219
1220 /*
1221 ====================
1222 Curl_RequireFile
1223
1224 Adds the given file to the list of requirements.
1225 ====================
1226 */
1227 void Curl_RequireFile(const char *filename)
1228 {
1229         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1230         req->next = requirements;
1231         strlcpy(req->filename, filename, sizeof(req->filename));
1232         requirements = req;
1233 }
1234
1235 /*
1236 ====================
1237 Curl_ClearRequirements
1238
1239 Clears the list of required files for playing on the current map.
1240 This should be called at every map change.
1241 ====================
1242 */
1243 void Curl_ClearRequirements()
1244 {
1245         const char *p;
1246         while(requirements)
1247         {
1248                 requirement *req = requirements;
1249                 requirements = requirements->next;
1250                 Z_Free(req);
1251         }
1252         p = sv_curl_serverpackages.string;
1253         Con_DPrintf("Require all of: %s\n", p);
1254         while(COM_ParseTokenConsole(&p))
1255         {
1256                 Con_DPrintf("Require: %s\n", com_token);
1257                 Curl_RequireFile(com_token);
1258         }
1259 }
1260
1261 /*
1262 ====================
1263 Curl_SendRequirements
1264
1265 Makes the current host_clients download all files he needs.
1266 This is done by sending him the following console commands:
1267
1268         curl --clear_autodownload
1269         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1270         curl --finish_autodownload
1271 ====================
1272 */
1273 void Curl_SendRequirements()
1274 {
1275         // for each requirement, find the pack name
1276         char sendbuffer[4096] = "";
1277         requirement *req;
1278         qboolean foundone = false;
1279
1280         for(req = requirements; req; req = req->next)
1281         {
1282                 const char *p;
1283                 const char *thispack = FS_WhichPack(req->filename);
1284                 const char *packurl;
1285
1286                 if(!thispack)
1287                         continue;
1288
1289                 p = strrchr(thispack, '/');
1290                 if(p)
1291                         thispack = p + 1;
1292
1293                 packurl = Curl_FindPackURL(thispack);
1294
1295                 if(packurl && *packurl && strcmp(packurl, "-"))
1296                 {
1297                         if(!foundone)
1298                                 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1299
1300                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1301                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1302                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1303                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1304                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1305                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1306                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1307                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1308
1309                         foundone = true;
1310                 }
1311         }
1312
1313         if(foundone)
1314                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1315
1316         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1317                 Host_ClientCommands("%s", sendbuffer);
1318         else
1319                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1320 }