libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5 #include "com_list.h"
   6 #include "image.h"
   7 #include "jpeg.h"
   8 #include "image_png.h"
   9
  10 static cvar_t cl_curl_maxdownloads = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
  11 static cvar_t cl_curl_maxspeed = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
  12 static cvar_t sv_curl_defaulturl = {CF_SERVER | CF_ARCHIVE, "sv_curl_defaulturl","", "default autodownload source URL"};
  13 static cvar_t sv_curl_serverpackages = {CF_SERVER | CF_ARCHIVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  14 static cvar_t sv_curl_maxspeed = {CF_SERVER | CF_ARCHIVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  15 static cvar_t cl_curl_enabled = {CF_CLIENT | CF_ARCHIVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  16 static cvar_t cl_curl_useragent = {CF_CLIENT, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  17 static cvar_t cl_curl_useragent_append = {CF_CLIENT, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  18 static cvar_t developer_curl = {CF_CLIENT | CF_SERVER, "developer_curl","0", "whether verbose curl output should be printed to stderr"};
  19
  20 /*
  21 =================================================================
  22
  23   Minimal set of definitions from libcurl
  24
  25   WARNING: for a matter of simplicity, several pointer types are
  26   casted to "void*", and most enumerated values are not included
  27
  28 =================================================================
  29 */
  30
  31 typedef struct CURL_s CURL;
  32 typedef struct CURLM_s CURLM;
  33 typedef struct curl_slist curl_slist;
  34 typedef enum
  35 {
  36         CURLE_OK = 0
  37 }
  38 CURLcode;
  39 typedef enum
  40 {
  41         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  42         CURLM_OK = 0
  43 }
  44 CURLMcode;
  45 #define CURL_GLOBAL_NOTHING 0
  46 #define CURL_GLOBAL_SSL 1
  47 #define CURL_GLOBAL_WIN32 2
  48 #define CURLOPTTYPE_LONG          0
  49 #define CURLOPTTYPE_OBJECTPOINT   10000
  50 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  51 #define CURLOPTTYPE_OFF_T         30000
  52 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  53 typedef enum
  54 {
  55         CINIT(WRITEDATA, OBJECTPOINT, 1),
  56         CINIT(URL,  OBJECTPOINT, 2),
  57         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  58         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  59         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  60         CINIT(REFERER, OBJECTPOINT, 16),
  61         CINIT(USERAGENT, OBJECTPOINT, 18),
  62         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  63         CINIT(LOW_SPEED_TIME, LONG, 20),
  64         CINIT(RESUME_FROM, LONG, 21),
  65         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  66         CINIT(VERBOSE, LONG, 41),
  67         CINIT(POST, LONG, 47),         /* HTTP POST method */
  68         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  69         CINIT(POSTFIELDSIZE, LONG, 60),
  70         CINIT(PRIVATE, OBJECTPOINT, 103),
  71         CINIT(PROTOCOLS, LONG, 181),
  72         CINIT(REDIR_PROTOCOLS, LONG, 182)
  73 }
  74 CURLoption;
  75 #define CURLPROTO_HTTP   (1<<0)
  76 #define CURLPROTO_HTTPS  (1<<1)
  77 #define CURLPROTO_FTP    (1<<2)
  78 typedef enum
  79 {
  80         CURLINFO_TEXT = 0,
  81         CURLINFO_HEADER_IN,    /* 1 */
  82         CURLINFO_HEADER_OUT,   /* 2 */
  83         CURLINFO_DATA_IN,      /* 3 */
  84         CURLINFO_DATA_OUT,     /* 4 */
  85         CURLINFO_SSL_DATA_IN,  /* 5 */
  86         CURLINFO_SSL_DATA_OUT, /* 6 */
  87         CURLINFO_END
  88 }
  89 curl_infotype;
  90 #define CURLINFO_STRING   0x100000
  91 #define CURLINFO_LONG     0x200000
  92 #define CURLINFO_DOUBLE   0x300000
  93 #define CURLINFO_SLIST    0x400000
  94 #define CURLINFO_MASK     0x0fffff
  95 #define CURLINFO_TYPEMASK 0xf00000
  96 typedef enum
  97 {
  98         CURLINFO_NONE, /* first, never use this */
  99         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
 100         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
 101         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
 102         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
 103         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
 104         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
 105         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 106         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 107         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 108         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 109         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 110         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 111         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 112         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 113         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 114         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 115         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 116         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 117         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 118         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 119         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 120         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 121         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 122         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 123         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 124         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 125         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 126 }
 127 CURLINFO;
 128
 129 typedef enum
 130 {
 131         CURLMSG_NONE, /* first, not used */
 132         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 133                                          the CURLcode of the transfer */
 134         CURLMSG_LAST
 135 }
 136 CURLMSG;
 137 typedef struct
 138 {
 139         CURLMSG msg;       /* what this message means */
 140         CURL *easy_handle; /* the handle it concerns */
 141         union
 142         {
 143                 void *whatever;    /* message-specific data */
 144                 CURLcode result;   /* return code for transfer */
 145         }
 146         data;
 147 }
 148 CURLMsg;
 149
 150 static void (*qcurl_global_init) (long flags);
 151 static void (*qcurl_global_cleanup) (void);
 152
 153 static CURL * (*qcurl_easy_init) (void);
 154 static void (*qcurl_easy_cleanup) (CURL *handle);
 155 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 156 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 157 static const char * (*qcurl_easy_strerror) (CURLcode);
 158
 159 static CURLM * (*qcurl_multi_init) (void);
 160 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 161 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 162 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 163 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 164 static void (*qcurl_multi_cleanup) (CURLM *);
 165 static const char * (*qcurl_multi_strerror) (CURLcode);
 166 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 167 static void (*qcurl_slist_free_all) (curl_slist *list);
 168
 169 static dllfunction_t curlfuncs[] =
 170 {
 171         {"curl_global_init",            (void **) &qcurl_global_init},
 172         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 173         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 174         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 175         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 176         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 177         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 178         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 179         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 180         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 181         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 182         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 183         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 184         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 185         {"curl_slist_append",           (void **) &qcurl_slist_append},
 186         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 187         {NULL, NULL}
 188 };
 189
 190 // Handle for CURL DLL
 191 static dllhandle_t curl_dll = NULL;
 192 // will be checked at many places to find out if qcurl calls are allowed
 193
 194 #define LOADTYPE_NONE 0
 195 #define LOADTYPE_PAK 1
 196 #define LOADTYPE_CACHEPIC 2
 197 #define LOADTYPE_SKINFRAME 3
 198
 199 void *curl_mutex = NULL;
 200
 201 typedef struct downloadinfo_s
 202 {
 203         char filename[MAX_OSPATH];
 204         char url[1024];
 205         char referer[256];
 206         qfile_t *stream;
 207         fs_offset_t startpos;
 208         CURL *curle;
 209         qbool started;
 210         int loadtype;
 211         size_t bytes_received; // for buffer
 212         double bytes_received_curl; // for throttling
 213         double bytes_sent_curl; // for throttling
 214         llist_t list;
 215         qbool forthismap;
 216         double maxspeed;
 217         curl_slist *slist; // http headers
 218
 219         unsigned char *buffer;
 220         size_t buffersize;
 221         curl_callback_t callback;
 222         void *callback_data;
 223
 224         const unsigned char *postbuf;
 225         size_t postbufsize;
 226         const char *post_content_type;
 227         const char *extraheaders;
 228 }
 229 downloadinfo;
 230 LIST_HEAD(downloads);
 231 static int numdownloads = 0;
 232
 233 static qbool noclear = false;
 234
 235 static int numdownloads_fail = 0;
 236 static int numdownloads_success = 0;
 237 static int numdownloads_added = 0;
 238 static char command_when_done[256] = "";
 239 static char command_when_error[256] = "";
 240
 241 /*
 242 ====================
 243 Curl_CommandWhenDone
 244
 245 Sets the command which is to be executed when the last download completes AND
 246 all downloads since last server connect ended with a successful status.
 247 Setting the command to NULL clears it.
 248 ====================
 249 */
 250 static void Curl_CommandWhenDone(const char *cmd)
 251 {
 252         if(!curl_dll)
 253                 return;
 254         if(cmd)
 255                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 256         else
 257                 *command_when_done = 0;
 258 }
 259
 260 /*
 261 FIXME
 262 Do not use yet. Not complete.
 263 Problem: what counts as an error?
 264 */
 265
 266 static void Curl_CommandWhenError(const char *cmd)
 267 {
 268         if(!curl_dll)
 269                 return;
 270         if(cmd)
 271                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 272         else
 273                 *command_when_error = 0;
 274 }
 275
 276 /*
 277 ====================
 278 Curl_Clear_forthismap
 279
 280 Clears the "will disconnect on failure" flags.
 281 ====================
 282 */
 283 void Curl_Clear_forthismap(void)
 284 {
 285         downloadinfo *di;
 286         if(noclear)
 287                 return;
 288         if (curl_mutex) Thread_LockMutex(curl_mutex);
 289         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 290                 di->forthismap = false;
 291         Curl_CommandWhenError(NULL);
 292         Curl_CommandWhenDone(NULL);
 293         numdownloads_fail = 0;
 294         numdownloads_success = 0;
 295         numdownloads_added = 0;
 296         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 297 }
 298
 299 /*
 300 ====================
 301 Curl_Have_forthismap
 302
 303 Returns true if a download needed for the current game is running.
 304 ====================
 305 */
 306 qbool Curl_Have_forthismap(void)
 307 {
 308         return numdownloads_added != 0;
 309 }
 310
 311 void Curl_Register_predownload(void)
 312 {
 313         if (curl_mutex) Thread_LockMutex(curl_mutex);
 314         Curl_CommandWhenDone("cl_begindownloads");
 315         Curl_CommandWhenError("cl_begindownloads");
 316         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 317 }
 318
 319 /*
 320 ====================
 321 Curl_CheckCommandWhenDone
 322
 323 Checks if a "done command" is to be executed.
 324 All downloads finished, at least one success since connect, no single failure
 325 -> execute the command.
 326 */
 327 static void Curl_CheckCommandWhenDone(void)
 328 {
 329         if(!curl_dll)
 330                 return;
 331         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 332         {
 333                 if(numdownloads_fail == 0)
 334                 {
 335                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 336                         Cbuf_AddText(cmd_local, "\n");
 337                         Cbuf_AddText(cmd_local, command_when_done);
 338                         Cbuf_AddText(cmd_local, "\n");
 339                 }
 340                 else
 341                 {
 342                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 343                         Cbuf_AddText(cmd_local, "\n");
 344                         Cbuf_AddText(cmd_local, command_when_error);
 345                         Cbuf_AddText(cmd_local, "\n");
 346                 }
 347                 Curl_Clear_forthismap();
 348         }
 349 }
 350
 351 /*
 352 ====================
 353 CURL_CloseLibrary
 354
 355 Load the cURL DLL
 356 ====================
 357 */
 358 static qbool CURL_OpenLibrary (void)
 359 {
 360         const char* dllnames [] =
 361         {
 362 #if defined(WIN32)
 363                 "libcurl-4.dll",
 364                 "libcurl-3.dll",
 365 #elif defined(MACOSX)
 366                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 367                 "libcurl.3.dylib", // Mac OS X Tiger
 368                 "libcurl.2.dylib", // Mac OS X Panther
 369 #else
 370                 "libcurl.so.4",
 371                 "libcurl.so.3",
 372                 "libcurl.so", // FreeBSD
 373 #endif
 374                 NULL
 375         };
 376
 377         // Already loaded?
 378         if (curl_dll)
 379                 return true;
 380
 381         // Load the DLL
 382         return Sys_LoadDependency (dllnames, &curl_dll, curlfuncs);
 383 }
 384
 385
 386 /*
 387 ====================
 388 CURL_CloseLibrary
 389
 390 Unload the cURL DLL
 391 ====================
 392 */
 393 static void CURL_CloseLibrary (void)
 394 {
 395         Sys_FreeLibrary (&curl_dll);
 396 }
 397
 398
 399 static CURLM *curlm = NULL;
 400 static double bytes_received = 0; // used for bandwidth throttling
 401 static double bytes_sent = 0; // used for bandwidth throttling
 402 static double curltime = 0;
 403
 404 /*
 405 ====================
 406 CURL_fwrite
 407
 408 fwrite-compatible function that writes the data to a file. libcurl can call
 409 this.
 410 ====================
 411 */
 412 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 413 {
 414         fs_offset_t ret = -1;
 415         size_t bytes = size * nmemb;
 416         downloadinfo *di = (downloadinfo *) vdi;
 417
 418         if(di->buffer)
 419         {
 420                 if(di->bytes_received + bytes <= di->buffersize)
 421                 {
 422                         memcpy(di->buffer + di->bytes_received, data, bytes);
 423                         ret = bytes;
 424                 }
 425                 // otherwise: buffer overrun, ret stays -1
 426         }
 427
 428         if(di->stream)
 429         {
 430                 ret = FS_Write(di->stream, data, bytes);
 431         }
 432
 433         di->bytes_received += bytes;
 434
 435         return ret;
 436         // Why not ret / nmemb?
 437         // Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
 438         // Yes, this is incompatible to fwrite(2).
 439 }
 440
 441 typedef enum
 442 {
 443         CURL_DOWNLOAD_SUCCESS = 0,
 444         CURL_DOWNLOAD_FAILED,
 445         CURL_DOWNLOAD_ABORTED,
 446         CURL_DOWNLOAD_SERVERERROR
 447 }
 448 CurlStatus;
 449
 450 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 451 {
 452         downloadinfo *di = (downloadinfo *) cbdata;
 453         switch(status)
 454         {
 455                 case CURLCBSTATUS_OK:
 456                         Con_DPrintf("Download of %s: OK\n", di->filename);
 457                         break;
 458                 case CURLCBSTATUS_FAILED:
 459                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 460                         break;
 461                 case CURLCBSTATUS_ABORTED:
 462                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 463                         break;
 464                 case CURLCBSTATUS_SERVERERROR:
 465                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 466                         break;
 467                 case CURLCBSTATUS_UNKNOWN:
 468                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 469                         break;
 470                 default:
 471                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 472                         break;
 473         }
 474 }
 475
 476 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 477 {
 478         curl_default_callback(status, length_received, buffer, cbdata);
 479 }
 480
 481 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
 482 {
 483         unsigned char *pixels = NULL;
 484         fs_offset_t filesize = 0;
 485         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
 486         if(data)
 487         {
 488                 int mip = 0;
 489                 if(!strcmp(content_type, "image/jpeg"))
 490                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 491                 else if(!strcmp(content_type, "image/png"))
 492                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 493                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
 494                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 495                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
 496                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 497                 else
 498                         Con_Printf("Did not detect content type: %s\n", content_type);
 499                 Mem_Free(data);
 500         }
 501         // do we call Image_MakeLinearColorsFromsRGB or not?
 502         return pixels;
 503 }
 504
 505 /*
 506 ====================
 507 Curl_EndDownload
 508
 509 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 510 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 511 code from libcurl, or 0, if another error has occurred.
 512 ====================
 513 */
 514 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 515 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
 516 {
 517         char content_type[64];
 518         qbool ok = false;
 519         if(!curl_dll)
 520                 return;
 521         switch(status)
 522         {
 523                 case CURL_DOWNLOAD_SUCCESS:
 524                         ok = true;
 525                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 526                         break;
 527                 case CURL_DOWNLOAD_FAILED:
 528                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 529                         break;
 530                 case CURL_DOWNLOAD_ABORTED:
 531                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 532                         break;
 533                 case CURL_DOWNLOAD_SERVERERROR:
 534                         // reopen to enforce it to have zero bytes again
 535                         if(di->stream)
 536                         {
 537                                 FS_Close(di->stream);
 538                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 539                         }
 540
 541                         if(di->callback)
 542                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 543                         break;
 544                 default:
 545                         if(di->callback)
 546                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 547                         break;
 548         }
 549         if(content_type_)
 550                 strlcpy(content_type, content_type_, sizeof(content_type));
 551         else
 552                 *content_type = 0;
 553
 554         if(di->curle)
 555         {
 556                 qcurl_multi_remove_handle(curlm, di->curle);
 557                 qcurl_easy_cleanup(di->curle);
 558                 if(di->slist)
 559                         qcurl_slist_free_all(di->slist);
 560         }
 561
 562         if(!di->callback && ok && !di->bytes_received)
 563         {
 564                 Con_Printf("ERROR: empty file\n");
 565                 ok = false;
 566         }
 567
 568         if(di->stream)
 569                 FS_Close(di->stream);
 570
 571 #define CLEAR_AND_RETRY() \
 572         do \
 573         { \
 574                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
 575                 FS_Close(di->stream); \
 576                 if(di->startpos && !di->callback) \
 577                 { \
 578                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
 579                         di->forthismap = false; \
 580                 } \
 581         } \
 582         while(0)
 583
 584         if(ok && di->loadtype == LOADTYPE_PAK)
 585         {
 586                 ok = FS_AddPack(di->filename, NULL, true);
 587                 if(!ok)
 588                         CLEAR_AND_RETRY();
 589         }
 590         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
 591         {
 592                 const char *p;
 593                 unsigned char *pixels = NULL;
 594
 595                 p = di->filename;
 596 #ifdef WE_ARE_EVIL
 597                 if(!strncmp(p, "dlcache/", 8))
 598                         p += 8;
 599 #endif
 600
 601                 pixels = decode_image(di, content_type);
 602                 if(pixels)
 603                         Draw_NewPic(p, image_width, image_height, pixels, TEXTYPE_BGRA, TEXF_ALPHA | TEXF_CLAMP);
 604                 else
 605                         CLEAR_AND_RETRY();
 606         }
 607         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
 608         {
 609                 const char *p;
 610                 unsigned char *pixels = NULL;
 611
 612                 p = di->filename;
 613 #ifdef WE_ARE_EVIL
 614                 if(!strncmp(p, "dlcache/", 8))
 615                         p += 8;
 616 #endif
 617
 618                 pixels = decode_image(di, content_type);
 619                 if(pixels)
 620                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, 0, 0, 0, false); // TODO what sRGB argument to put here?
 621                 else
 622                         CLEAR_AND_RETRY();
 623         }
 624
 625         List_Delete(&di->list);
 626
 627         --numdownloads;
 628         if(di->forthismap)
 629         {
 630                 if(ok)
 631                         ++numdownloads_success;
 632                 else
 633                         ++numdownloads_fail;
 634         }
 635         Z_Free(di);
 636 }
 637
 638 /*
 639 ====================
 640 CleanURL
 641
 642 Returns a "cleaned up" URL for display (to strip login data)
 643 ====================
 644 */
 645 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 646 {
 647         const char *p, *q, *r;
 648
 649         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 650         p = strstr(url, "://");
 651         if(p)
 652         {
 653                 q = strchr(p + 3, '@');
 654                 if(q)
 655                 {
 656                         r = strchr(p + 3, '/');
 657                         if(!r || q < r)
 658                         {
 659                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 660                                 return urlbuf;
 661                         }
 662                 }
 663         }
 664
 665         return url;
 666 }
 667
 668 /*
 669 ====================
 670 CheckPendingDownloads
 671
 672 checks if there are free download slots to start new downloads in.
 673 To not start too many downloads at once, only one download is added at a time,
 674 up to a maximum number of cl_curl_maxdownloads are running.
 675 ====================
 676 */
 677 static void CheckPendingDownloads(void)
 678 {
 679         const char *h;
 680         char urlbuf[1024];
 681         char vabuf[1024];
 682         if(!curl_dll)
 683                 return;
 684         if(numdownloads < cl_curl_maxdownloads.integer)
 685         {
 686                 downloadinfo *di;
 687                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
 688                 {
 689                         if(!di->started)
 690                         {
 691                                 if(!di->buffer)
 692                                 {
 693                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 694
 695                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 696                                         if(!di->stream)
 697                                         {
 698                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 699                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
 700                                                 return;
 701                                         }
 702                                         FS_Seek(di->stream, 0, SEEK_END);
 703                                         di->startpos = FS_Tell(di->stream);
 704
 705                                         if(di->startpos > 0)
 706                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 707                                         Con_Print("...\n");
 708                                 }
 709                                 else
 710                                 {
 711                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 712                                         di->startpos = 0;
 713                                 }
 714
 715                                 di->curle = qcurl_easy_init();
 716                                 di->slist = NULL;
 717                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 718                                 if(cl_curl_useragent.integer)
 719                                 {
 720                                         const char *ua
 721 #ifdef HTTP_USER_AGENT
 722                                                 = HTTP_USER_AGENT;
 723 #else
 724                                                 = engineversion;
 725 #endif
 726                                         if(!ua)
 727                                                 ua = "";
 728                                         if(*cl_curl_useragent_append.string)
 729                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 730                                                         ua,
 731                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 732                                                                 ? " "
 733                                                                 : "",
 734                                                         cl_curl_useragent_append.string);
 735                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 736                                 }
 737                                 else
 738                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 739                                 if(developer_curl.integer)
 740                                         qcurl_easy_setopt(di->curle, CURLOPT_VERBOSE, (long) 1);
 741                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 742                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 743                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 744                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 745                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 746                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 747                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 748                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 749                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 750                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 751                                 {
 752                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 753                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 754                                 }
 755                                 if(di->post_content_type)
 756                                 {
 757                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 758                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 759                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 760                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 761                                 }
 762
 763                                 // parse extra headers into slist
 764                                 // \n separated list!
 765                                 h = di->extraheaders;
 766                                 while(h)
 767                                 {
 768                                         const char *hh = strchr(h, '\n');
 769                                         if(hh)
 770                                         {
 771                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 772                                                 memcpy(buf, h, hh - h);
 773                                                 buf[hh - h] = 0;
 774                                                 di->slist = qcurl_slist_append(di->slist, buf);
 775                                                 h = hh + 1;
 776                                         }
 777                                         else
 778                                         {
 779                                                 di->slist = qcurl_slist_append(di->slist, h);
 780                                                 h = NULL;
 781                                         }
 782                                 }
 783
 784                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 785
 786                                 qcurl_multi_add_handle(curlm, di->curle);
 787                                 di->started = true;
 788                                 ++numdownloads;
 789                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 790                                         break;
 791                         }
 792                 }
 793         }
 794 }
 795
 796 /*
 797 ====================
 798 Curl_Init
 799
 800 this function MUST be called before using anything else in this file.
 801 On Win32, this must be called AFTER WSAStartup has been done!
 802 ====================
 803 */
 804 void Curl_Init(void)
 805 {
 806         CURL_OpenLibrary();
 807         if(!curl_dll)
 808                 return;
 809         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 810         qcurl_global_init(CURL_GLOBAL_NOTHING);
 811         curlm = qcurl_multi_init();
 812 }
 813
 814 /*
 815 ====================
 816 Curl_Shutdown
 817
 818 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 819 ====================
 820 */
 821 void Curl_ClearRequirements(void);
 822 void Curl_Shutdown(void)
 823 {
 824         if(!curl_dll)
 825                 return;
 826         Curl_ClearRequirements();
 827         Curl_CancelAll();
 828         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 829         CURL_CloseLibrary();
 830         curl_dll = NULL;
 831 }
 832
 833 /*
 834 ====================
 835 Curl_Find
 836
 837 Finds the internal information block for a download given by file name.
 838 ====================
 839 */
 840 static downloadinfo *Curl_Find(const char *filename)
 841 {
 842         downloadinfo *di;
 843         if(!curl_dll)
 844                 return NULL;
 845         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 846                 if(!strcasecmp(di->filename, filename))
 847                         return di;
 848         return NULL;
 849 }
 850
 851 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 852 {
 853         downloadinfo *di, *ndi;
 854         if(!curl_dll)
 855                 return;
 856         List_For_Each_Entry_Safe(di, ndi, &downloads, downloadinfo, list)
 857         {
 858                 if(di->callback == callback && di->callback_data == cbdata)
 859                 {
 860                         di->callback = curl_quiet_callback; // do NOT call the callback
 861                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
 862                 }
 863         }
 864 }
 865
 866 /*
 867 ====================
 868 Curl_Begin
 869
 870 Starts a download of a given URL to the file name portion of this URL (or name
 871 if given) in the "dlcache/" folder.
 872 ====================
 873 */
 874 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 875 {
 876         if(buf)
 877                 if(loadtype != LOADTYPE_NONE)
 878                         Host_Error("Curl_Begin: loadtype and buffer are both set");
 879
 880         if(!curl_dll || !cl_curl_enabled.integer)
 881         {
 882                 return false;
 883         }
 884         else
 885         {
 886                 char fn[MAX_OSPATH];
 887                 char urlbuf[1024];
 888                 const char *p, *q;
 889                 size_t length;
 890                 downloadinfo *di;
 891
 892                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 893                 p = strchr(URL, ':');
 894                 if(p)
 895                 {
 896                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 897                         {
 898                                 char addressstring[128];
 899                                 *addressstring = 0;
 900                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 901                                 q = strchr(addressstring, ':');
 902                                 if(!q)
 903                                         q = addressstring + strlen(addressstring);
 904                                 if(*addressstring)
 905                                 {
 906                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 907                                         URL = urlbuf;
 908                                 }
 909                         }
 910                 }
 911
 912                 // Note: This extraction of the file name portion is NOT entirely correct.
 913                 //
 914                 // It does the following:
 915                 //
 916                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 917                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 918                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 919                 //
 920                 // However, I'd like to keep this "buggy" behavior so that PHP script
 921                 // authors can write download scripts without having to enable
 922                 // AcceptPathInfo on Apache. They just have to ensure that their script
 923                 // can be called with such a "fake" path name like
 924                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 925                 //
 926                 // By the way, such PHP scripts should either send the file or a
 927                 // "Location:" redirect; PHP code example:
 928                 //
 929                 //   header("Location: http://www.example.com/");
 930                 //
 931                 // By the way, this will set User-Agent to something like
 932                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 933                 // dp://serverhost:serverport/ so you can filter on this; an example
 934                 // httpd log file line might be:
 935                 //
 936                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 937
 938                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 939
 940                 if(buf)
 941                 {
 942                         if(!name)
 943                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 944                 }
 945                 else
 946                 {
 947                         if(!name)
 948                         {
 949                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 950                                 p = strrchr(name, '/');
 951                                 p = p ? (p+1) : name;
 952                                 q = strchr(p, '?');
 953                                 length = q ? (size_t)(q - p) : strlen(p);
 954                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 955                         }
 956                         else
 957                         {
 958                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
 959                         }
 960
 961                         name = fn; // make it point back
 962
 963                         // already downloading the file?
 964                         {
 965                                 downloadinfo *existingdownloadinfo = Curl_Find(fn);
 966                                 if(existingdownloadinfo)
 967                                 {
 968                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(existingdownloadinfo->url, urlbuf, sizeof(urlbuf)));
 969
 970                                         // however, if it was not for this map yet...
 971                                         if(forthismap && !existingdownloadinfo->forthismap)
 972                                         {
 973                                                 existingdownloadinfo->forthismap = true;
 974                                                 // this "fakes" a download attempt so the client will wait for
 975                                                 // the download to finish and then reconnect
 976                                                 ++numdownloads_added;
 977                                         }
 978
 979                                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 980                                         return false;
 981                                 }
 982                         }
 983
 984                         if(FS_FileExists(fn))
 985                         {
 986                                 if(loadtype == LOADTYPE_PAK)
 987                                 {
 988                                         qbool already_loaded;
 989                                         if(FS_AddPack(fn, &already_loaded, true))
 990                                         {
 991                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 992                                                 if(already_loaded)
 993                                                         Con_DPrintf("(pak was already loaded)\n");
 994                                                 else
 995                                                 {
 996                                                         if(forthismap)
 997                                                         {
 998                                                                 ++numdownloads_added;
 999                                                                 ++numdownloads_success;
1000                                                         }
1001                                                 }
1002
1003                                                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1004                                                 return false;
1005                                         }
1006                                         else
1007                                         {
1008                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1009                                                 if(f)
1010                                                 {
1011                                                         char b[4] = {0};
1012                                                         FS_Read(f, b, sizeof(b)); // no "-1", I will use memcmp
1013
1014                                                         if(memcmp(b, "PK\x03\x04", 4) && memcmp(b, "PACK", 4))
1015                                                         {
1016                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1017                                                                 FS_Close(f);
1018                                                                 f = FS_OpenRealFile(fn, "wb", false);
1019                                                                 if(f)
1020                                                                         FS_Close(f);
1021                                                         }
1022                                                         else
1023                                                         {
1024                                                                 // OK
1025                                                                 FS_Close(f);
1026                                                         }
1027                                                 }
1028                                         }
1029                                 }
1030                                 else
1031                                 {
1032                                         // never resume these
1033                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1034                                         if(f)
1035                                                 FS_Close(f);
1036                                 }
1037                         }
1038                 }
1039
1040                 // if we get here, we actually want to download... so first verify the
1041                 // URL scheme (so one can't read local files using file://)
1042                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1043                 {
1044                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1045                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1046                         return false;
1047                 }
1048
1049                 if(forthismap)
1050                         ++numdownloads_added;
1051                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1052                 strlcpy(di->filename, name, sizeof(di->filename));
1053                 strlcpy(di->url, URL, sizeof(di->url));
1054                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1055                 di->forthismap = forthismap;
1056                 di->stream = NULL;
1057                 di->startpos = 0;
1058                 di->curle = NULL;
1059                 di->started = false;
1060                 di->loadtype = loadtype;
1061                 di->maxspeed = maxspeed;
1062                 di->bytes_received = 0;
1063                 di->bytes_received_curl = 0;
1064                 di->bytes_sent_curl = 0;
1065                 di->extraheaders = extraheaders;
1066                 di->buffer = buf;
1067                 di->buffersize = bufsize;
1068                 if(callback == NULL)
1069                 {
1070                         di->callback = curl_default_callback;
1071                         di->callback_data = di;
1072                 }
1073                 else
1074                 {
1075                         di->callback = callback;
1076                         di->callback_data = cbdata;
1077                 }
1078
1079                 if(post_content_type)
1080                 {
1081                         di->post_content_type = post_content_type;
1082                         di->postbuf = postbuf;
1083                         di->postbufsize = postbufsize;
1084                 }
1085                 else
1086                 {
1087                         di->post_content_type = NULL;
1088                         di->postbuf = NULL;
1089                         di->postbufsize = 0;
1090                 }
1091
1092                 List_Add(&di->list, &downloads);
1093
1094                 if (curl_mutex)
1095                         Thread_UnlockMutex(curl_mutex);
1096
1097                 return true;
1098         }
1099 }
1100
1101 qbool Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qbool forthismap)
1102 {
1103         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1104 }
1105 qbool Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1106 {
1107         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1108 }
1109 qbool Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1110 {
1111         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1112 }
1113
1114 /*
1115 ====================
1116 Curl_Frame
1117
1118 call this regularily as this will always download as much as possible without
1119 blocking.
1120 ====================
1121 */
1122 void Curl_Frame(void)
1123 {
1124         double maxspeed;
1125         downloadinfo *di;
1126
1127         noclear = false;
1128
1129         if(!cl_curl_enabled.integer)
1130                 return;
1131
1132         if(!curl_dll)
1133                 return;
1134
1135         if (curl_mutex) Thread_LockMutex(curl_mutex);
1136
1137         Curl_CheckCommandWhenDone();
1138
1139         if(List_Is_Empty(&downloads))
1140         {
1141                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1142                 return;
1143         }
1144
1145         if(host.realtime < curltime) // throttle
1146         {
1147                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1148                 return;
1149         }
1150
1151         {
1152                 int remaining;
1153                 CURLMcode mc;
1154
1155                 do
1156                 {
1157                         mc = qcurl_multi_perform(curlm, &remaining);
1158                 }
1159                 while(mc == CURLM_CALL_MULTI_PERFORM);
1160
1161                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1162                 {
1163                         double b = 0;
1164                         if(di->curle)
1165                         {
1166                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1167                                 bytes_sent += (b - di->bytes_sent_curl);
1168                                 di->bytes_sent_curl = b;
1169                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1170                                 bytes_sent += (b - di->bytes_received_curl);
1171                                 di->bytes_received_curl = b;
1172                         }
1173                 }
1174
1175                 for(;;)
1176                 {
1177                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1178                         if(!msg)
1179                                 break;
1180                         if(msg->msg == CURLMSG_DONE)
1181                         {
1182                                 const char *ct = NULL;
1183                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1184                                 CURLcode result;
1185                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1186                                 result = msg->data.result;
1187                                 if(result)
1188                                 {
1189                                         failed = CURL_DOWNLOAD_FAILED;
1190                                 }
1191                                 else
1192                                 {
1193                                         long code;
1194                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1195                                         switch(code / 100)
1196                                         {
1197                                                 case 4: // e.g. 404?
1198                                                 case 5: // e.g. 500?
1199                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1200                                                         result = (CURLcode) code;
1201                                                         break;
1202                                         }
1203                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1204                                 }
1205
1206                                 Curl_EndDownload(di, failed, result, ct);
1207                         }
1208                 }
1209         }
1210
1211         CheckPendingDownloads();
1212
1213         // when will we curl the next time?
1214         // we will wait a bit to ensure our download rate is kept.
1215         // we now know that realtime >= curltime... so set up a new curltime
1216
1217         // use the slowest allowing download to derive the maxspeed... this CAN
1218         // be done better, but maybe later
1219         maxspeed = cl_curl_maxspeed.value;
1220         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1221                 if(di->maxspeed > 0)
1222                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1223                                 maxspeed = di->maxspeed;
1224
1225         if(maxspeed > 0)
1226         {
1227                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1228                 curltime = host.realtime + bytes / (maxspeed * 1024.0);
1229                 bytes_sent = 0;
1230                 bytes_received = 0;
1231         }
1232         else
1233                 curltime = host.realtime;
1234
1235         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1236 }
1237
1238 /*
1239 ====================
1240 Curl_CancelAll
1241
1242 Stops ALL downloads.
1243 ====================
1244 */
1245 void Curl_CancelAll(void)
1246 {
1247         if(!curl_dll)
1248                 return;
1249
1250         if (curl_mutex) Thread_LockMutex(curl_mutex);
1251
1252         while(!List_Is_Empty(&downloads))
1253         {
1254                 Curl_EndDownload(List_First_Entry(&downloads, downloadinfo, list), CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1255                 // INVARIANT: downloads will point to the next download after that!
1256         }
1257
1258         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1259 }
1260
1261 /*
1262 ====================
1263 Curl_Running
1264
1265 returns true if there is a download running.
1266 ====================
1267 */
1268 qbool Curl_Running(void)
1269 {
1270         if(!curl_dll)
1271                 return false;
1272
1273         return !List_Is_Empty(&downloads);
1274 }
1275
1276 /*
1277 ====================
1278 Curl_GetDownloadAmount
1279
1280 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1281 for the given download.
1282 ====================
1283 */
1284 static double Curl_GetDownloadAmount(downloadinfo *di)
1285 {
1286         if(!curl_dll)
1287                 return -2;
1288         if(di->curle)
1289         {
1290                 double length;
1291                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1292                 if(length > 0)
1293                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1294                 else
1295                         return 0;
1296         }
1297         else
1298                 return -1;
1299 }
1300
1301 /*
1302 ====================
1303 Curl_GetDownloadSpeed
1304
1305 returns the speed of the given download in bytes per second
1306 ====================
1307 */
1308 static double Curl_GetDownloadSpeed(downloadinfo *di)
1309 {
1310         if(!curl_dll)
1311                 return -2;
1312         if(di->curle)
1313         {
1314                 double speed;
1315                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1316                 return speed;
1317         }
1318         else
1319                 return -1;
1320 }
1321
1322 /*
1323 ====================
1324 Curl_Info_f
1325
1326 prints the download list
1327 ====================
1328 */
1329 // TODO rewrite using Curl_GetDownloadInfo?
1330 static void Curl_Info_f(cmd_state_t *cmd)
1331 {
1332         downloadinfo *di;
1333         char urlbuf[1024];
1334         if(!curl_dll)
1335                 return;
1336         if(Curl_Running())
1337         {
1338                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1339                 Con_Print("Currently running downloads:\n");
1340                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1341                 {
1342                         double speed, percent;
1343                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1344                         percent = 100.0 * Curl_GetDownloadAmount(di);
1345                         speed = Curl_GetDownloadSpeed(di);
1346                         if(percent >= 0)
1347                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1348                         else
1349                                 Con_Print("(queued)\n");
1350                 }
1351                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1352         }
1353         else
1354         {
1355                 Con_Print("No downloads running.\n");
1356         }
1357 }
1358
1359 /*
1360 ====================
1361 Curl_Curl_f
1362
1363 implements the "curl" console command
1364
1365 curl --info
1366 curl --cancel
1367 curl --cancel filename
1368 curl url
1369
1370 For internal use:
1371
1372 curl [--pak] [--forthismap] [--for filename filename...] url
1373         --pak: after downloading, load the package into the virtual file system
1374         --for filename...: only download of at least one of the named files is missing
1375         --forthismap: don't reconnect on failure
1376
1377 curl --clear_autodownload
1378         clears the download success/failure counters
1379
1380 curl --finish_autodownload
1381         if at least one download has been started, disconnect and drop to the menu
1382         once the last download completes successfully, reconnect to the current server
1383 ====================
1384 */
1385 static void Curl_Curl_f(cmd_state_t *cmd)
1386 {
1387         double maxspeed = 0;
1388         int i;
1389         int end;
1390         int loadtype = LOADTYPE_NONE;
1391         qbool forthismap = false;
1392         const char *url;
1393         const char *name = 0;
1394
1395         if(!curl_dll)
1396         {
1397                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1398                 return;
1399         }
1400
1401         if(!cl_curl_enabled.integer)
1402         {
1403                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1404                 return;
1405         }
1406
1407         if(Cmd_Argc(cmd) < 2)
1408         {
1409                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1410                 return;
1411         }
1412
1413         url = Cmd_Argv(cmd, Cmd_Argc(cmd) - 1);
1414         end = Cmd_Argc(cmd);
1415
1416         for(i = 1; i != end; ++i)
1417         {
1418                 const char *a = Cmd_Argv(cmd, i);
1419                 if(!strcmp(a, "--info"))
1420                 {
1421                         Curl_Info_f(cmd);
1422                         return;
1423                 }
1424                 else if(!strcmp(a, "--cancel"))
1425                 {
1426                         if(i == end - 1) // last argument
1427                                 Curl_CancelAll();
1428                         else
1429                         {
1430                                 downloadinfo *di = Curl_Find(url);
1431                                 if(di)
1432                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1433                                 else
1434                                         Con_Print("download not found\n");
1435                         }
1436                         return;
1437                 }
1438                 else if(!strcmp(a, "--pak"))
1439                 {
1440                         loadtype = LOADTYPE_PAK;
1441                 }
1442                 else if(!strcmp(a, "--cachepic"))
1443                 {
1444                         loadtype = LOADTYPE_CACHEPIC;
1445                 }
1446                 else if(!strcmp(a, "--skinframe"))
1447                 {
1448                         loadtype = LOADTYPE_SKINFRAME;
1449                 }
1450                 else if(!strcmp(a, "--for")) // must be last option
1451                 {
1452                         for(i = i + 1; i != end - 1; ++i)
1453                         {
1454                                 if(!FS_FileExists(Cmd_Argv(cmd, i)))
1455                                         goto needthefile; // why can't I have a "double break"?
1456                         }
1457                         // if we get here, we have all the files...
1458                         return;
1459                 }
1460                 else if(!strcmp(a, "--forthismap"))
1461                 {
1462                         forthismap = true;
1463                 }
1464                 else if(!strcmp(a, "--as"))
1465                 {
1466                         if(i < end - 1)
1467                         {
1468                                 ++i;
1469                                 name = Cmd_Argv(cmd, i);
1470                         }
1471                 }
1472                 else if(!strcmp(a, "--clear_autodownload"))
1473                 {
1474                         // mark all running downloads as "not for this map", so if they
1475                         // fail, it does not matter
1476                         Curl_Clear_forthismap();
1477                         return;
1478                 }
1479                 else if(!strcmp(a, "--finish_autodownload"))
1480                 {
1481                         if(numdownloads_added)
1482                         {
1483                                 char donecommand[256];
1484                                 if(cls.netcon)
1485                                 {
1486                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1487                                         {
1488                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1489                                                 Curl_CommandWhenDone(donecommand);
1490                                                 noclear = true;
1491                                                 CL_Disconnect();
1492                                                 noclear = false;
1493                                                 Curl_CheckCommandWhenDone();
1494                                         }
1495                                         else
1496                                                 Curl_Register_predownload();
1497                                 }
1498                         }
1499                         return;
1500                 }
1501                 else if(!strncmp(a, "--maxspeed=", 11))
1502                 {
1503                         maxspeed = atof(a + 11);
1504                 }
1505                 else if(*a == '-')
1506                 {
1507                         Con_Printf("curl: invalid option %s\n", a);
1508                         // but we ignore the option
1509                 }
1510         }
1511
1512 needthefile:
1513         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1514 }
1515
1516 /*
1517 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1518 {
1519         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1520         Z_Free(buffer);
1521 }
1522
1523 void Curl_CurlCat_f(cmd_state_t *cmd)
1524 {
1525         unsigned char *buf;
1526         const char *url = Cmd_Argv(cmd, 1);
1527         buf = Z_Malloc(16384);
1528         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1529 }
1530 */
1531
1532 /*
1533 ====================
1534 Curl_Init_Commands
1535
1536 loads the commands and cvars this library uses
1537 ====================
1538 */
1539 void Curl_Init_Commands(void)
1540 {
1541         Cvar_RegisterVariable (&cl_curl_enabled);
1542         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1543         Cvar_RegisterVariable (&cl_curl_maxspeed);
1544         Cvar_RegisterVariable (&sv_curl_defaulturl);
1545         Cvar_RegisterVariable (&sv_curl_serverpackages);
1546         Cvar_RegisterVariable (&sv_curl_maxspeed);
1547         Cvar_RegisterVariable (&cl_curl_useragent);
1548         Cvar_RegisterVariable (&cl_curl_useragent_append);
1549         Cvar_RegisterVariable (&developer_curl);
1550         Cmd_AddCommand(CF_CLIENT | CF_CLIENT_FROM_SERVER, "curl", Curl_Curl_f, "download data from an URL and add to search path");
1551         //Cmd_AddCommand(cmd_local, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1552 }
1553
1554 /*
1555 ====================
1556 Curl_GetDownloadInfo
1557
1558 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1559 The number of elements in the array is returned in int *nDownloads.
1560 const char **additional_info may be set to a string of additional user
1561 information, or to NULL if no such display shall occur. The returned
1562 array must be freed later using Z_Free.
1563 ====================
1564 */
1565 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1566 {
1567         int i;
1568         downloadinfo *di;
1569         Curl_downloadinfo_t *downinfo;
1570
1571         if(!curl_dll)
1572         {
1573                 *nDownloads = 0;
1574                 if(additional_info)
1575                         *additional_info = NULL;
1576                 return NULL;
1577         }
1578
1579         if (curl_mutex) Thread_LockMutex(curl_mutex);
1580
1581         i = 0;
1582         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1583                 ++i;
1584
1585         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1586         i = 0;
1587         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1588         {
1589                 // do not show infobars for background downloads
1590                 if(developer.integer <= 0)
1591                         if(di->buffer)
1592                                 continue;
1593                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1594                 if(di->curle)
1595                 {
1596                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1597                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1598                         downinfo[i].queued = false;
1599                 }
1600                 else
1601                 {
1602                         downinfo[i].queued = true;
1603                 }
1604                 ++i;
1605         }
1606
1607         if(additional_info)
1608         {
1609                 // TODO: can I clear command_when_done as soon as the first download fails?
1610                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1611                 {
1612                         if(!strncmp(command_when_done, "connect ", 8))
1613                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1614                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1615                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1616                         else
1617                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1618                         *additional_info = addinfo;
1619                 }
1620                 else
1621                         *additional_info = NULL;
1622         }
1623
1624         *nDownloads = i;
1625         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1626         return downinfo;
1627 }
1628
1629
1630 /*
1631 ====================
1632 Curl_FindPackURL
1633
1634 finds the URL where to find a given package.
1635
1636 For this, it reads a file "curl_urls.txt" of the following format:
1637
1638         data*.pk3       -
1639         revdm*.pk3      http://revdm/downloads/are/here/
1640         *                       http://any/other/stuff/is/here/
1641
1642 The URLs should end in /. If not, downloads will still work, but the cached files
1643 can't be just put into the data directory with the same download configuration
1644 (you might want to do this if you want to tag downloaded files from your
1645 server, but you should not). "-" means "don't download".
1646
1647 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1648 location instead.
1649
1650 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1651 this file for obvious reasons.
1652 ====================
1653 */
1654 static const char *Curl_FindPackURL(const char *filename)
1655 {
1656         static char foundurl[1024]; // invoked only by server
1657         fs_offset_t filesize;
1658         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1659         if(buf && filesize)
1660         {
1661                 // read lines of format "pattern url"
1662                 char *p = buf;
1663                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1664                 qbool eof = false;
1665
1666                 pattern = p;
1667                 while(!eof)
1668                 {
1669                         switch(*p)
1670                         {
1671                                 case 0:
1672                                         eof = true;
1673                                         // fallthrough
1674                                 case '\n':
1675                                 case '\r':
1676                                         if(pattern && url && patternend)
1677                                         {
1678                                                 if(!urlend)
1679                                                         urlend = p;
1680                                                 *patternend = 0;
1681                                                 *urlend = 0;
1682                                                 if(matchpattern(filename, pattern, true))
1683                                                 {
1684                                                         strlcpy(foundurl, url, sizeof(foundurl));
1685                                                         Z_Free(buf);
1686                                                         return foundurl;
1687                                                 }
1688                                         }
1689                                         pattern = NULL;
1690                                         patternend = NULL;
1691                                         url = NULL;
1692                                         urlend = NULL;
1693                                         break;
1694                                 case ' ':
1695                                 case '\t':
1696                                         if(pattern && !patternend)
1697                                                 patternend = p;
1698                                         else if(url && !urlend)
1699                                                 urlend = p;
1700                                         break;
1701                                 default:
1702                                         if(!pattern)
1703                                                 pattern = p;
1704                                         else if(pattern && patternend && !url)
1705                                                 url = p;
1706                                         break;
1707                         }
1708                         ++p;
1709                 }
1710         }
1711         if(buf)
1712                 Z_Free(buf);
1713         return sv_curl_defaulturl.string;
1714 }
1715
1716 typedef struct requirement_s
1717 {
1718         struct requirement_s *next;
1719         char filename[MAX_OSPATH];
1720 }
1721 requirement;
1722 static requirement *requirements = NULL;
1723
1724
1725 /*
1726 ====================
1727 Curl_RequireFile
1728
1729 Adds the given file to the list of requirements.
1730 ====================
1731 */
1732 void Curl_RequireFile(const char *filename)
1733 {
1734         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1735         req->next = requirements;
1736         strlcpy(req->filename, filename, sizeof(req->filename));
1737         requirements = req;
1738 }
1739
1740 /*
1741 ====================
1742 Curl_ClearRequirements
1743
1744 Clears the list of required files for playing on the current map.
1745 This should be called at every map change.
1746 ====================
1747 */
1748 void Curl_ClearRequirements(void)
1749 {
1750         while(requirements)
1751         {
1752                 requirement *req = requirements;
1753                 requirements = requirements->next;
1754                 Z_Free(req);
1755         }
1756 }
1757
1758 /*
1759 ====================
1760 Curl_SendRequirements
1761
1762 Makes the current host_clients download all files he needs.
1763 This is done by sending him the following console commands:
1764
1765         curl --clear_autodownload
1766         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1767         curl --finish_autodownload
1768 ====================
1769 */
1770 static qbool Curl_SendRequirement(const char *filename, qbool foundone, char *sendbuffer, size_t sendbuffer_len)
1771 {
1772         const char *p;
1773         const char *thispack = FS_WhichPack(filename);
1774         const char *packurl;
1775
1776         if(!thispack || !*thispack)
1777                 return false;
1778
1779         p = strrchr(thispack, '/');
1780         if(p)
1781                 thispack = p + 1;
1782
1783         packurl = Curl_FindPackURL(thispack);
1784
1785         if(packurl && *packurl && strcmp(packurl, "-"))
1786         {
1787                 if(!foundone)
1788                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1789
1790                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1791                 strlcat(sendbuffer, thispack, sendbuffer_len);
1792                 if(sv_curl_maxspeed.value > 0)
1793                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1794                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1795                 strlcat(sendbuffer, filename, sendbuffer_len);
1796                 strlcat(sendbuffer, " ", sendbuffer_len);
1797                 strlcat(sendbuffer, packurl, sendbuffer_len);
1798                 strlcat(sendbuffer, thispack, sendbuffer_len);
1799                 strlcat(sendbuffer, "\n", sendbuffer_len);
1800
1801                 return true;
1802         }
1803
1804         return false;
1805 }
1806 void Curl_SendRequirements(void)
1807 {
1808         // for each requirement, find the pack name
1809         char sendbuffer[4096] = "";
1810         requirement *req;
1811         qbool foundone = false;
1812         const char *p;
1813
1814         for(req = requirements; req; req = req->next)
1815                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1816
1817         p = sv_curl_serverpackages.string;
1818         while(COM_ParseToken_Simple(&p, false, false, true))
1819                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1820
1821         if(foundone)
1822                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1823
1824         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1825                 SV_ClientCommands("%s", sendbuffer);
1826         else
1827                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1828 }