libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5 #include "com_list.h"
   6 #include "image.h"
   7 #include "jpeg.h"
   8 #include "image_png.h"
   9
  10 static cvar_t cl_curl_maxdownloads = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
  11 static cvar_t cl_curl_maxspeed = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
  12 static cvar_t sv_curl_defaulturl = {CF_SERVER | CF_ARCHIVE, "sv_curl_defaulturl","", "default autodownload source URL"};
  13 static cvar_t sv_curl_serverpackages = {CF_SERVER | CF_ARCHIVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  14 static cvar_t sv_curl_maxspeed = {CF_SERVER | CF_ARCHIVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  15 static cvar_t cl_curl_enabled = {CF_CLIENT | CF_ARCHIVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  16 static cvar_t cl_curl_useragent = {CF_CLIENT, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  17 static cvar_t cl_curl_useragent_append = {CF_CLIENT, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  18 static cvar_t developer_curl = {CF_CLIENT | CF_SERVER, "developer_curl","0", "whether verbose curl output should be printed to stderr"};
  19
  20 /*
  21 =================================================================
  22
  23   Minimal set of definitions from libcurl
  24
  25   WARNING: for a matter of simplicity, several pointer types are
  26   casted to "void*", and most enumerated values are not included
  27
  28 =================================================================
  29 */
  30
  31 typedef struct CURL_s CURL;
  32 typedef struct CURLM_s CURLM;
  33 typedef struct curl_slist curl_slist;
  34 typedef enum
  35 {
  36         CURLE_OK = 0
  37 }
  38 CURLcode;
  39 typedef enum
  40 {
  41         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  42         CURLM_OK = 0
  43 }
  44 CURLMcode;
  45 #define CURL_GLOBAL_NOTHING 0
  46 #define CURL_GLOBAL_SSL 1
  47 #define CURL_GLOBAL_WIN32 2
  48 #define CURLOPTTYPE_LONG          0
  49 #define CURLOPTTYPE_OBJECTPOINT   10000
  50 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  51 #define CURLOPTTYPE_OFF_T         30000
  52 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  53 typedef enum
  54 {
  55         CINIT(WRITEDATA, OBJECTPOINT, 1),
  56         CINIT(URL,  OBJECTPOINT, 2),
  57         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  58         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  59         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  60         CINIT(REFERER, OBJECTPOINT, 16),
  61         CINIT(USERAGENT, OBJECTPOINT, 18),
  62         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  63         CINIT(LOW_SPEED_TIME, LONG, 20),
  64         CINIT(RESUME_FROM, LONG, 21),
  65         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  66         CINIT(VERBOSE, LONG, 41),
  67         CINIT(POST, LONG, 47),         /* HTTP POST method */
  68         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  69         CINIT(POSTFIELDSIZE, LONG, 60),
  70         CINIT(PRIVATE, OBJECTPOINT, 103),
  71         CINIT(PROTOCOLS, LONG, 181),
  72         CINIT(REDIR_PROTOCOLS, LONG, 182)
  73 }
  74 CURLoption;
  75 #define CURLPROTO_HTTP   (1<<0)
  76 #define CURLPROTO_HTTPS  (1<<1)
  77 #define CURLPROTO_FTP    (1<<2)
  78 typedef enum
  79 {
  80         CURLINFO_TEXT = 0,
  81         CURLINFO_HEADER_IN,    /* 1 */
  82         CURLINFO_HEADER_OUT,   /* 2 */
  83         CURLINFO_DATA_IN,      /* 3 */
  84         CURLINFO_DATA_OUT,     /* 4 */
  85         CURLINFO_SSL_DATA_IN,  /* 5 */
  86         CURLINFO_SSL_DATA_OUT, /* 6 */
  87         CURLINFO_END
  88 }
  89 curl_infotype;
  90 #define CURLINFO_STRING   0x100000
  91 #define CURLINFO_LONG     0x200000
  92 #define CURLINFO_DOUBLE   0x300000
  93 #define CURLINFO_SLIST    0x400000
  94 #define CURLINFO_MASK     0x0fffff
  95 #define CURLINFO_TYPEMASK 0xf00000
  96 typedef enum
  97 {
  98         CURLINFO_NONE, /* first, never use this */
  99         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
 100         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
 101         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
 102         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
 103         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
 104         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
 105         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 106         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 107         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 108         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 109         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 110         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 111         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 112         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 113         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 114         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 115         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 116         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 117         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 118         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 119         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 120         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 121         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 122         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 123         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 124         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 125         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 126 }
 127 CURLINFO;
 128
 129 typedef enum
 130 {
 131         CURLMSG_NONE, /* first, not used */
 132         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 133                                          the CURLcode of the transfer */
 134         CURLMSG_LAST
 135 }
 136 CURLMSG;
 137 typedef struct
 138 {
 139         CURLMSG msg;       /* what this message means */
 140         CURL *easy_handle; /* the handle it concerns */
 141         union
 142         {
 143                 void *whatever;    /* message-specific data */
 144                 CURLcode result;   /* return code for transfer */
 145         }
 146         data;
 147 }
 148 CURLMsg;
 149
 150 static void (*qcurl_global_init) (long flags);
 151 static void (*qcurl_global_cleanup) (void);
 152
 153 static CURL * (*qcurl_easy_init) (void);
 154 static void (*qcurl_easy_cleanup) (CURL *handle);
 155 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 156 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 157 static const char * (*qcurl_easy_strerror) (CURLcode);
 158
 159 static CURLM * (*qcurl_multi_init) (void);
 160 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 161 static CURLMcode (*qcurl_multi_poll) (CURLM *multi_handle, void*, unsigned int extra_nfds, int timeout_ms, int *ret);
 162 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 163 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 164 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 165 static void (*qcurl_multi_cleanup) (CURLM *);
 166 static const char * (*qcurl_multi_strerror) (CURLcode);
 167 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 168 static void (*qcurl_slist_free_all) (curl_slist *list);
 169
 170 static dllfunction_t curlfuncs[] =
 171 {
 172         {"curl_global_init",            (void **) &qcurl_global_init},
 173         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 174         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 175         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 176         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 177         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 178         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 179         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 180         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 181         {"curl_multi_poll",             (void **) &qcurl_multi_poll},
 182         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 183         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 184         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 185         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 186         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 187         {"curl_slist_append",           (void **) &qcurl_slist_append},
 188         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 189         {NULL, NULL}
 190 };
 191
 192 // Handle for CURL DLL
 193 static dllhandle_t curl_dll = NULL;
 194 // will be checked at many places to find out if qcurl calls are allowed
 195
 196 #define LOADTYPE_NONE 0
 197 #define LOADTYPE_PAK 1
 198 #define LOADTYPE_CACHEPIC 2
 199 #define LOADTYPE_SKINFRAME 3
 200
 201 void *curl_mutex = NULL;
 202
 203 typedef struct downloadinfo_s
 204 {
 205         char filename[MAX_OSPATH];
 206         char url[1024];
 207         char referer[256];
 208         qfile_t *stream;
 209         fs_offset_t startpos;
 210         CURL *curle;
 211         qbool started;
 212         int loadtype;
 213         size_t bytes_received; // for buffer
 214         double bytes_received_curl; // for throttling
 215         double bytes_sent_curl; // for throttling
 216         llist_t list;
 217         qbool forthismap;
 218         double maxspeed;
 219         curl_slist *slist; // http headers
 220
 221         unsigned char *buffer;
 222         size_t buffersize;
 223         curl_callback_t callback;
 224         void *callback_data;
 225
 226         const unsigned char *postbuf;
 227         size_t postbufsize;
 228         const char *post_content_type;
 229         const char *extraheaders;
 230 }
 231 downloadinfo;
 232 LIST_HEAD(downloads);
 233 static int numdownloads = 0;
 234
 235 static qbool noclear = false;
 236
 237 static int numdownloads_fail = 0;
 238 static int numdownloads_success = 0;
 239 static int numdownloads_added = 0;
 240 static char command_when_done[256] = "";
 241 static char command_when_error[256] = "";
 242
 243 /*
 244 ====================
 245 Curl_CommandWhenDone
 246
 247 Sets the command which is to be executed when the last download completes AND
 248 all downloads since last server connect ended with a successful status.
 249 Setting the command to NULL clears it.
 250 ====================
 251 */
 252 static void Curl_CommandWhenDone(const char *cmd)
 253 {
 254         if(!curl_dll)
 255                 return;
 256         if(cmd)
 257                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 258         else
 259                 *command_when_done = 0;
 260 }
 261
 262 /*
 263 FIXME
 264 Do not use yet. Not complete.
 265 Problem: what counts as an error?
 266 */
 267
 268 static void Curl_CommandWhenError(const char *cmd)
 269 {
 270         if(!curl_dll)
 271                 return;
 272         if(cmd)
 273                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 274         else
 275                 *command_when_error = 0;
 276 }
 277
 278 /*
 279 ====================
 280 Curl_Clear_forthismap
 281
 282 Clears the "will disconnect on failure" flags.
 283 ====================
 284 */
 285 void Curl_Clear_forthismap(void)
 286 {
 287         downloadinfo *di;
 288         if(noclear)
 289                 return;
 290         if (curl_mutex) Thread_LockMutex(curl_mutex);
 291         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 292                 di->forthismap = false;
 293         Curl_CommandWhenError(NULL);
 294         Curl_CommandWhenDone(NULL);
 295         numdownloads_fail = 0;
 296         numdownloads_success = 0;
 297         numdownloads_added = 0;
 298         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 299 }
 300
 301 /*
 302 ====================
 303 Curl_Have_forthismap
 304
 305 Returns true if a download needed for the current game is running.
 306 ====================
 307 */
 308 qbool Curl_Have_forthismap(void)
 309 {
 310         return numdownloads_added != 0;
 311 }
 312
 313 void Curl_Register_predownload(void)
 314 {
 315         if (curl_mutex) Thread_LockMutex(curl_mutex);
 316         Curl_CommandWhenDone("cl_begindownloads");
 317         Curl_CommandWhenError("cl_begindownloads");
 318         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 319 }
 320
 321 /*
 322 ====================
 323 Curl_CheckCommandWhenDone
 324
 325 Checks if a "done command" is to be executed.
 326 All downloads finished, at least one success since connect, no single failure
 327 -> execute the command.
 328 */
 329 static void Curl_CheckCommandWhenDone(void)
 330 {
 331         if(!curl_dll)
 332                 return;
 333         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 334         {
 335                 if(numdownloads_fail == 0)
 336                 {
 337                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 338                         Cbuf_AddText(cmd_local, "\n");
 339                         Cbuf_AddText(cmd_local, command_when_done);
 340                         Cbuf_AddText(cmd_local, "\n");
 341                 }
 342                 else
 343                 {
 344                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 345                         Cbuf_AddText(cmd_local, "\n");
 346                         Cbuf_AddText(cmd_local, command_when_error);
 347                         Cbuf_AddText(cmd_local, "\n");
 348                 }
 349                 Curl_Clear_forthismap();
 350         }
 351 }
 352
 353 /*
 354 ====================
 355 CURL_CloseLibrary
 356
 357 Load the cURL DLL
 358 ====================
 359 */
 360 static qbool CURL_OpenLibrary (void)
 361 {
 362         const char* dllnames [] =
 363         {
 364 #if defined(WIN32)
 365                 "libcurl-4.dll",
 366                 "libcurl-3.dll",
 367 #elif defined(MACOSX)
 368                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 369                 "libcurl.3.dylib", // Mac OS X Tiger
 370                 "libcurl.2.dylib", // Mac OS X Panther
 371 #else
 372                 "libcurl.so.4",
 373                 "libcurl.so.3",
 374                 "libcurl.so", // FreeBSD
 375 #endif
 376                 NULL
 377         };
 378
 379         // Already loaded?
 380         if (curl_dll)
 381                 return true;
 382
 383         // Load the DLL
 384         return Sys_LoadDependency (dllnames, &curl_dll, curlfuncs);
 385 }
 386
 387
 388 /*
 389 ====================
 390 CURL_CloseLibrary
 391
 392 Unload the cURL DLL
 393 ====================
 394 */
 395 static void CURL_CloseLibrary (void)
 396 {
 397         Sys_FreeLibrary (&curl_dll);
 398 }
 399
 400
 401 static CURLM *curlm = NULL;
 402 static double bytes_received = 0; // used for bandwidth throttling
 403 static double bytes_sent = 0; // used for bandwidth throttling
 404 static double curltime = 0;
 405
 406 /*
 407 ====================
 408 CURL_fwrite
 409
 410 fwrite-compatible function that writes the data to a file. libcurl can call
 411 this.
 412 ====================
 413 */
 414 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 415 {
 416         fs_offset_t ret = -1;
 417         size_t bytes = size * nmemb;
 418         downloadinfo *di = (downloadinfo *) vdi;
 419
 420         if(di->buffer)
 421         {
 422                 if(di->bytes_received + bytes <= di->buffersize)
 423                 {
 424                         memcpy(di->buffer + di->bytes_received, data, bytes);
 425                         ret = bytes;
 426                 }
 427                 // otherwise: buffer overrun, ret stays -1
 428         }
 429
 430         if(di->stream)
 431         {
 432                 ret = FS_Write(di->stream, data, bytes);
 433         }
 434
 435         di->bytes_received += bytes;
 436
 437         //Con_Printf("CURL_fwrite callback timestamp: %f bytes: %ld\n", host.realtime, ret);
 438
 439         return ret;
 440         // Why not ret / nmemb?
 441         // Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
 442         // Yes, this is incompatible to fwrite(2).
 443 }
 444
 445 typedef enum
 446 {
 447         CURL_DOWNLOAD_SUCCESS = 0,
 448         CURL_DOWNLOAD_FAILED,
 449         CURL_DOWNLOAD_ABORTED,
 450         CURL_DOWNLOAD_SERVERERROR
 451 }
 452 CurlStatus;
 453
 454 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 455 {
 456         downloadinfo *di = (downloadinfo *) cbdata;
 457         switch(status)
 458         {
 459                 case CURLCBSTATUS_OK:
 460                         Con_DPrintf("Download of %s: OK\n", di->filename);
 461                         break;
 462                 case CURLCBSTATUS_FAILED:
 463                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 464                         break;
 465                 case CURLCBSTATUS_ABORTED:
 466                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 467                         break;
 468                 case CURLCBSTATUS_SERVERERROR:
 469                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 470                         break;
 471                 case CURLCBSTATUS_UNKNOWN:
 472                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 473                         break;
 474                 default:
 475                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 476                         break;
 477         }
 478 }
 479
 480 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 481 {
 482         curl_default_callback(status, length_received, buffer, cbdata);
 483 }
 484
 485 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
 486 {
 487         unsigned char *pixels = NULL;
 488         fs_offset_t filesize = 0;
 489         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
 490         if(data)
 491         {
 492                 int mip = 0;
 493                 if(!strcmp(content_type, "image/jpeg"))
 494                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 495                 else if(!strcmp(content_type, "image/png"))
 496                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 497                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
 498                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 499                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
 500                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 501                 else
 502                         Con_Printf("Did not detect content type: %s\n", content_type);
 503                 Mem_Free(data);
 504         }
 505         // do we call Image_MakeLinearColorsFromsRGB or not?
 506         return pixels;
 507 }
 508
 509 /*
 510 ====================
 511 Curl_EndDownload
 512
 513 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 514 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 515 code from libcurl, or 0, if another error has occurred.
 516 ====================
 517 */
 518 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 519 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
 520 {
 521         char content_type[64];
 522         qbool ok = false;
 523         if(!curl_dll)
 524                 return;
 525         switch(status)
 526         {
 527                 case CURL_DOWNLOAD_SUCCESS:
 528                         ok = true;
 529                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 530                         break;
 531                 case CURL_DOWNLOAD_FAILED:
 532                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 533                         break;
 534                 case CURL_DOWNLOAD_ABORTED:
 535                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 536                         break;
 537                 case CURL_DOWNLOAD_SERVERERROR:
 538                         // reopen to enforce it to have zero bytes again
 539                         if(di->stream)
 540                         {
 541                                 FS_Close(di->stream);
 542                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 543                         }
 544
 545                         if(di->callback)
 546                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 547                         break;
 548                 default:
 549                         if(di->callback)
 550                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 551                         break;
 552         }
 553         if(content_type_)
 554                 strlcpy(content_type, content_type_, sizeof(content_type));
 555         else
 556                 *content_type = 0;
 557
 558         if(di->curle)
 559         {
 560                 qcurl_multi_remove_handle(curlm, di->curle);
 561                 qcurl_easy_cleanup(di->curle);
 562                 if(di->slist)
 563                         qcurl_slist_free_all(di->slist);
 564         }
 565
 566         if(!di->callback && ok && !di->bytes_received)
 567         {
 568                 Con_Printf("ERROR: empty file\n");
 569                 ok = false;
 570         }
 571
 572         if(di->stream)
 573                 FS_Close(di->stream);
 574
 575 #define CLEAR_AND_RETRY() \
 576         do \
 577         { \
 578                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
 579                 FS_Close(di->stream); \
 580                 if(di->startpos && !di->callback) \
 581                 { \
 582                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
 583                         di->forthismap = false; \
 584                 } \
 585         } \
 586         while(0)
 587
 588         if(ok && di->loadtype == LOADTYPE_PAK)
 589         {
 590                 ok = FS_AddPack(di->filename, NULL, true, true);
 591                 if(!ok)
 592                         CLEAR_AND_RETRY();
 593         }
 594         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
 595         {
 596                 const char *p;
 597                 unsigned char *pixels = NULL;
 598
 599                 p = di->filename;
 600 #ifdef WE_ARE_EVIL
 601                 if(!strncmp(p, "dlcache/", 8))
 602                         p += 8;
 603 #endif
 604
 605                 pixels = decode_image(di, content_type);
 606                 if(pixels)
 607                         Draw_NewPic(p, image_width, image_height, pixels, TEXTYPE_BGRA, TEXF_ALPHA | TEXF_CLAMP);
 608                 else
 609                         CLEAR_AND_RETRY();
 610         }
 611         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
 612         {
 613                 const char *p;
 614                 unsigned char *pixels = NULL;
 615
 616                 p = di->filename;
 617 #ifdef WE_ARE_EVIL
 618                 if(!strncmp(p, "dlcache/", 8))
 619                         p += 8;
 620 #endif
 621
 622                 pixels = decode_image(di, content_type);
 623                 if(pixels)
 624                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, 0, 0, 0, false); // TODO what sRGB argument to put here?
 625                 else
 626                         CLEAR_AND_RETRY();
 627         }
 628
 629         List_Delete(&di->list);
 630
 631         --numdownloads;
 632         if(di->forthismap)
 633         {
 634                 if(ok)
 635                         ++numdownloads_success;
 636                 else
 637                         ++numdownloads_fail;
 638         }
 639         Z_Free(di);
 640 }
 641
 642 /*
 643 ====================
 644 CleanURL
 645
 646 Returns a "cleaned up" URL for display (to strip login data)
 647 ====================
 648 */
 649 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 650 {
 651         const char *p, *q, *r;
 652
 653         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 654         p = strstr(url, "://");
 655         if(p)
 656         {
 657                 q = strchr(p + 3, '@');
 658                 if(q)
 659                 {
 660                         r = strchr(p + 3, '/');
 661                         if(!r || q < r)
 662                         {
 663                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 664                                 return urlbuf;
 665                         }
 666                 }
 667         }
 668
 669         return url;
 670 }
 671
 672 /*
 673 ====================
 674 CheckPendingDownloads
 675
 676 checks if there are free download slots to start new downloads in.
 677 To not start too many downloads at once, only one download is added at a time,
 678 up to a maximum number of cl_curl_maxdownloads are running.
 679 ====================
 680 */
 681 static void CheckPendingDownloads(void)
 682 {
 683         const char *h;
 684         char urlbuf[1024];
 685         char vabuf[1024];
 686         if(!curl_dll)
 687                 return;
 688         if(numdownloads < cl_curl_maxdownloads.integer)
 689         {
 690                 downloadinfo *di;
 691                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
 692                 {
 693                         if(!di->started)
 694                         {
 695                                 if(!di->buffer)
 696                                 {
 697                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 698
 699                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 700                                         if(!di->stream)
 701                                         {
 702                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 703                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
 704                                                 return;
 705                                         }
 706                                         FS_Seek(di->stream, 0, SEEK_END);
 707                                         di->startpos = FS_Tell(di->stream);
 708
 709                                         if(di->startpos > 0)
 710                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 711                                         Con_Print("...\n");
 712                                 }
 713                                 else
 714                                 {
 715                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 716                                         di->startpos = 0;
 717                                 }
 718
 719                                 di->curle = qcurl_easy_init();
 720                                 di->slist = NULL;
 721                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 722                                 if(cl_curl_useragent.integer)
 723                                 {
 724                                         const char *ua
 725 #ifdef HTTP_USER_AGENT
 726                                                 = HTTP_USER_AGENT;
 727 #else
 728                                                 = engineversion;
 729 #endif
 730                                         if(!ua)
 731                                                 ua = "";
 732                                         if(*cl_curl_useragent_append.string)
 733                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 734                                                         ua,
 735                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 736                                                                 ? " "
 737                                                                 : "",
 738                                                         cl_curl_useragent_append.string);
 739                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 740                                 }
 741                                 else
 742                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 743                                 if(developer_curl.integer)
 744                                         qcurl_easy_setopt(di->curle, CURLOPT_VERBOSE, (long) 1);
 745                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 746                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 747                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 748                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 749                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 750                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 751                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 752                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 753                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 754                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 755                                 {
 756                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 757                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 758                                 }
 759                                 if(di->post_content_type)
 760                                 {
 761                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 762                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 763                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 764                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 765                                 }
 766
 767                                 // parse extra headers into slist
 768                                 // \n separated list!
 769                                 h = di->extraheaders;
 770                                 while(h)
 771                                 {
 772                                         const char *hh = strchr(h, '\n');
 773                                         if(hh)
 774                                         {
 775                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 776                                                 memcpy(buf, h, hh - h);
 777                                                 buf[hh - h] = 0;
 778                                                 di->slist = qcurl_slist_append(di->slist, buf);
 779                                                 h = hh + 1;
 780                                         }
 781                                         else
 782                                         {
 783                                                 di->slist = qcurl_slist_append(di->slist, h);
 784                                                 h = NULL;
 785                                         }
 786                                 }
 787
 788                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 789
 790                                 qcurl_multi_add_handle(curlm, di->curle);
 791                                 di->started = true;
 792                                 ++numdownloads;
 793                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 794                                         break;
 795                         }
 796                 }
 797         }
 798 }
 799
 800 /*
 801 ====================
 802 Curl_Init
 803
 804 this function MUST be called before using anything else in this file.
 805 On Win32, this must be called AFTER WSAStartup has been done!
 806 ====================
 807 */
 808 void Curl_Init(void)
 809 {
 810         CURL_OpenLibrary();
 811         if(!curl_dll)
 812                 return;
 813         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 814         qcurl_global_init(CURL_GLOBAL_SSL);
 815         curlm = qcurl_multi_init();
 816 }
 817
 818 /*
 819 ====================
 820 Curl_Shutdown
 821
 822 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 823 ====================
 824 */
 825 void Curl_ClearRequirements(void);
 826 void Curl_Shutdown(void)
 827 {
 828         if(!curl_dll)
 829                 return;
 830         Curl_ClearRequirements();
 831         Curl_CancelAll();
 832         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 833         CURL_CloseLibrary();
 834         curl_dll = NULL;
 835 }
 836
 837 /*
 838 ====================
 839 Curl_Find
 840
 841 Finds the internal information block for a download given by file name.
 842 ====================
 843 */
 844 static downloadinfo *Curl_Find(const char *filename)
 845 {
 846         downloadinfo *di;
 847         if(!curl_dll)
 848                 return NULL;
 849         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 850                 if(!strcasecmp(di->filename, filename))
 851                         return di;
 852         return NULL;
 853 }
 854
 855 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 856 {
 857         downloadinfo *di, *ndi;
 858         if(!curl_dll)
 859                 return;
 860         List_For_Each_Entry_Safe(di, ndi, &downloads, downloadinfo, list)
 861         {
 862                 if(di->callback == callback && di->callback_data == cbdata)
 863                 {
 864                         di->callback = curl_quiet_callback; // do NOT call the callback
 865                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
 866                 }
 867         }
 868 }
 869
 870 /*
 871 ====================
 872 Curl_Begin
 873
 874 Starts a download of a given URL to the file name portion of this URL (or name
 875 if given) in the "dlcache/" folder.
 876 ====================
 877 */
 878 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 879 {
 880         if(buf)
 881                 if(loadtype != LOADTYPE_NONE)
 882                         Host_Error("Curl_Begin: loadtype and buffer are both set");
 883
 884         if(!curl_dll || !cl_curl_enabled.integer)
 885         {
 886                 return false;
 887         }
 888         else
 889         {
 890                 char fn[MAX_OSPATH];
 891                 char urlbuf[1024];
 892                 const char *p, *q;
 893                 size_t length;
 894                 downloadinfo *di;
 895
 896                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 897                 p = strchr(URL, ':');
 898                 if(p)
 899                 {
 900                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 901                         {
 902                                 char addressstring[128];
 903                                 *addressstring = 0;
 904                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 905                                 q = strchr(addressstring, ':');
 906                                 if(!q)
 907                                         q = addressstring + strlen(addressstring);
 908                                 if(*addressstring)
 909                                 {
 910                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 911                                         URL = urlbuf;
 912                                 }
 913                         }
 914                 }
 915
 916                 // Note: This extraction of the file name portion is NOT entirely correct.
 917                 //
 918                 // It does the following:
 919                 //
 920                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 921                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 922                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 923                 //
 924                 // However, I'd like to keep this "buggy" behavior so that PHP script
 925                 // authors can write download scripts without having to enable
 926                 // AcceptPathInfo on Apache. They just have to ensure that their script
 927                 // can be called with such a "fake" path name like
 928                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 929                 //
 930                 // By the way, such PHP scripts should either send the file or a
 931                 // "Location:" redirect; PHP code example:
 932                 //
 933                 //   header("Location: http://www.example.com/");
 934                 //
 935                 // By the way, this will set User-Agent to something like
 936                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 937                 // dp://serverhost:serverport/ so you can filter on this; an example
 938                 // httpd log file line might be:
 939                 //
 940                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 941
 942                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 943
 944                 if(buf)
 945                 {
 946                         if(!name)
 947                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 948                 }
 949                 else
 950                 {
 951                         if(!name)
 952                         {
 953                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 954                                 p = strrchr(name, '/');
 955                                 p = p ? (p+1) : name;
 956                                 q = strchr(p, '?');
 957                                 length = q ? (size_t)(q - p) : strlen(p);
 958                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 959                         }
 960                         else
 961                         {
 962                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
 963                         }
 964
 965                         name = fn; // make it point back
 966
 967                         // already downloading the file?
 968                         {
 969                                 downloadinfo *existingdownloadinfo = Curl_Find(fn);
 970                                 if(existingdownloadinfo)
 971                                 {
 972                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(existingdownloadinfo->url, urlbuf, sizeof(urlbuf)));
 973
 974                                         // however, if it was not for this map yet...
 975                                         if(forthismap && !existingdownloadinfo->forthismap)
 976                                         {
 977                                                 existingdownloadinfo->forthismap = true;
 978                                                 // this "fakes" a download attempt so the client will wait for
 979                                                 // the download to finish and then reconnect
 980                                                 ++numdownloads_added;
 981                                         }
 982
 983                                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 984                                         return false;
 985                                 }
 986                         }
 987
 988                         if(FS_FileExists(fn))
 989                         {
 990                                 if(loadtype == LOADTYPE_PAK)
 991                                 {
 992                                         qbool already_loaded;
 993                                         if(FS_AddPack(fn, &already_loaded, true, true))
 994                                         {
 995                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 996                                                 if(already_loaded)
 997                                                         Con_DPrintf("(pak was already loaded)\n");
 998                                                 else
 999                                                 {
1000                                                         if(forthismap)
1001                                                         {
1002                                                                 ++numdownloads_added;
1003                                                                 ++numdownloads_success;
1004                                                         }
1005                                                 }
1006
1007                                                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1008                                                 return false;
1009                                         }
1010                                         else
1011                                         {
1012                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1013                                                 if(f)
1014                                                 {
1015                                                         char b[4] = {0};
1016                                                         FS_Read(f, b, sizeof(b)); // no "-1", I will use memcmp
1017
1018                                                         if(memcmp(b, "PK\x03\x04", 4) && memcmp(b, "PACK", 4))
1019                                                         {
1020                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1021                                                                 FS_Close(f);
1022                                                                 f = FS_OpenRealFile(fn, "wb", false);
1023                                                                 if(f)
1024                                                                         FS_Close(f);
1025                                                         }
1026                                                         else
1027                                                         {
1028                                                                 // OK
1029                                                                 FS_Close(f);
1030                                                         }
1031                                                 }
1032                                         }
1033                                 }
1034                                 else
1035                                 {
1036                                         // never resume these
1037                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1038                                         if(f)
1039                                                 FS_Close(f);
1040                                 }
1041                         }
1042                 }
1043
1044                 // if we get here, we actually want to download... so first verify the
1045                 // URL scheme (so one can't read local files using file://)
1046                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1047                 {
1048                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1049                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1050                         return false;
1051                 }
1052
1053                 if(forthismap)
1054                         ++numdownloads_added;
1055                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1056                 strlcpy(di->filename, name, sizeof(di->filename));
1057                 strlcpy(di->url, URL, sizeof(di->url));
1058                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1059                 di->forthismap = forthismap;
1060                 di->stream = NULL;
1061                 di->startpos = 0;
1062                 di->curle = NULL;
1063                 di->started = false;
1064                 di->loadtype = loadtype;
1065                 di->maxspeed = maxspeed;
1066                 di->bytes_received = 0;
1067                 di->bytes_received_curl = 0;
1068                 di->bytes_sent_curl = 0;
1069                 di->extraheaders = extraheaders;
1070                 di->buffer = buf;
1071                 di->buffersize = bufsize;
1072                 if(callback == NULL)
1073                 {
1074                         di->callback = curl_default_callback;
1075                         di->callback_data = di;
1076                 }
1077                 else
1078                 {
1079                         di->callback = callback;
1080                         di->callback_data = cbdata;
1081                 }
1082
1083                 if(post_content_type)
1084                 {
1085                         di->post_content_type = post_content_type;
1086                         di->postbuf = postbuf;
1087                         di->postbufsize = postbufsize;
1088                 }
1089                 else
1090                 {
1091                         di->post_content_type = NULL;
1092                         di->postbuf = NULL;
1093                         di->postbufsize = 0;
1094                 }
1095
1096                 List_Add(&di->list, &downloads);
1097
1098                 if (curl_mutex)
1099                         Thread_UnlockMutex(curl_mutex);
1100
1101                 return true;
1102         }
1103 }
1104
1105 qbool Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qbool forthismap)
1106 {
1107         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1108 }
1109 qbool Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1110 {
1111         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1112 }
1113 qbool Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1114 {
1115         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1116 }
1117
1118 /*
1119 ====================
1120 Curl_Frame
1121
1122 call this regularily as this will always download as much as possible without
1123 blocking.
1124 ====================
1125 */
1126 void Curl_Frame(void)
1127 {
1128         double maxspeed;
1129         downloadinfo *di;
1130
1131         noclear = false;
1132
1133         if(!cl_curl_enabled.integer)
1134                 return;
1135
1136         if(!curl_dll)
1137                 return;
1138
1139         if (curl_mutex) Thread_LockMutex(curl_mutex);
1140
1141         Curl_CheckCommandWhenDone();
1142
1143         if(List_Is_Empty(&downloads))
1144         {
1145                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1146                 return;
1147         }
1148
1149         if(host.realtime < curltime) // throttle
1150         {
1151                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1152                 return;
1153         }
1154
1155         {
1156                 int remaining;
1157                 CURLMcode mc;
1158
1159                 do
1160                 {
1161                         mc = qcurl_multi_perform(curlm, &remaining);
1162                 }
1163                 while(mc == CURLM_CALL_MULTI_PERFORM);
1164
1165                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1166                 {
1167                         double b = 0;
1168                         if(di->curle)
1169                         {
1170                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1171                                 bytes_sent += (b - di->bytes_sent_curl);
1172                                 di->bytes_sent_curl = b;
1173                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1174                                 bytes_sent += (b - di->bytes_received_curl);
1175                                 di->bytes_received_curl = b;
1176                         }
1177                 }
1178
1179                 for(;;)
1180                 {
1181                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1182                         if(!msg)
1183                                 break;
1184                         if(msg->msg == CURLMSG_DONE)
1185                         {
1186                                 const char *ct = NULL;
1187                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1188                                 CURLcode result;
1189                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1190                                 result = msg->data.result;
1191                                 if(result)
1192                                 {
1193                                         failed = CURL_DOWNLOAD_FAILED;
1194                                 }
1195                                 else
1196                                 {
1197                                         long code;
1198                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1199                                         switch(code / 100)
1200                                         {
1201                                                 case 4: // e.g. 404?
1202                                                 case 5: // e.g. 500?
1203                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1204                                                         result = (CURLcode) code;
1205                                                         break;
1206                                         }
1207                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1208                                 }
1209
1210                                 Curl_EndDownload(di, failed, result, ct);
1211                         }
1212                 }
1213         }
1214
1215         CheckPendingDownloads();
1216
1217         // when will we curl the next time?
1218         // we will wait a bit to ensure our download rate is kept.
1219         // we now know that realtime >= curltime... so set up a new curltime
1220
1221         // use the slowest allowing download to derive the maxspeed... this CAN
1222         // be done better, but maybe later
1223         maxspeed = cl_curl_maxspeed.value;
1224         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1225                 if(di->maxspeed > 0)
1226                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1227                                 maxspeed = di->maxspeed;
1228
1229         if(maxspeed > 0)
1230         {
1231                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1232                 curltime = host.realtime + bytes / (maxspeed * 1024.0);
1233                 bytes_sent = 0;
1234                 bytes_received = 0;
1235         }
1236         else
1237                 curltime = host.realtime;
1238
1239         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1240 }
1241
1242 /*
1243 ====================
1244 Curl_Select
1245
1246 Sleeps until there's some transfer progress or a timeout is reached,
1247 unfortunately the timeout is only in milliseconds.
1248 This allows good throughput even at very low FPS.
1249 ====================
1250 */
1251 void Curl_Select(double *microseconds)
1252 {
1253         if (List_Is_Empty(&downloads))
1254                 return;
1255         if (qcurl_multi_poll(curlm, NULL, 0, *microseconds / 1000, NULL) == CURLM_OK)
1256                 *microseconds = 0; // either we finished waiting or a transfer progressed
1257         else
1258                 Con_Print("There's an emergency going on!\nIt's still going on!\nMaybe you need to upgrade libcurl?\n");
1259 }
1260
1261 /*
1262 ====================
1263 Curl_CancelAll
1264
1265 Stops ALL downloads.
1266 ====================
1267 */
1268 void Curl_CancelAll(void)
1269 {
1270         if(!curl_dll)
1271                 return;
1272
1273         if (curl_mutex) Thread_LockMutex(curl_mutex);
1274
1275         while(!List_Is_Empty(&downloads))
1276         {
1277                 Curl_EndDownload(List_First_Entry(&downloads, downloadinfo, list), CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1278                 // INVARIANT: downloads will point to the next download after that!
1279         }
1280
1281         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1282 }
1283
1284 /*
1285 ====================
1286 Curl_Running
1287
1288 returns true if there is a download running.
1289 ====================
1290 */
1291 qbool Curl_Running(void)
1292 {
1293         if(!curl_dll)
1294                 return false;
1295
1296         return !List_Is_Empty(&downloads);
1297 }
1298
1299 /*
1300 ====================
1301 Curl_GetDownloadAmount
1302
1303 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1304 for the given download.
1305 ====================
1306 */
1307 static double Curl_GetDownloadAmount(downloadinfo *di)
1308 {
1309         if(!curl_dll)
1310                 return -2;
1311         if(di->curle)
1312         {
1313                 double length;
1314                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1315                 if(length > 0)
1316                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1317                 else
1318                         return 0;
1319         }
1320         else
1321                 return -1;
1322 }
1323
1324 /*
1325 ====================
1326 Curl_GetDownloadSpeed
1327
1328 returns the speed of the given download in bytes per second
1329 ====================
1330 */
1331 static double Curl_GetDownloadSpeed(downloadinfo *di)
1332 {
1333         if(!curl_dll)
1334                 return -2;
1335         if(di->curle)
1336         {
1337                 double speed;
1338                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1339                 return speed;
1340         }
1341         else
1342                 return -1;
1343 }
1344
1345 /*
1346 ====================
1347 Curl_Info_f
1348
1349 prints the download list
1350 ====================
1351 */
1352 // TODO rewrite using Curl_GetDownloadInfo?
1353 static void Curl_Info_f(cmd_state_t *cmd)
1354 {
1355         downloadinfo *di;
1356         char urlbuf[1024];
1357         if(!curl_dll)
1358                 return;
1359         if(Curl_Running())
1360         {
1361                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1362                 Con_Print("Currently running downloads:\n");
1363                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1364                 {
1365                         double speed, percent;
1366                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1367                         percent = 100.0 * Curl_GetDownloadAmount(di);
1368                         speed = Curl_GetDownloadSpeed(di);
1369                         if(percent >= 0)
1370                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1371                         else
1372                                 Con_Print("(queued)\n");
1373                 }
1374                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1375         }
1376         else
1377         {
1378                 Con_Print("No downloads running.\n");
1379         }
1380 }
1381
1382 /*
1383 ====================
1384 Curl_Curl_f
1385
1386 implements the "curl" console command
1387
1388 curl --info
1389 curl --cancel
1390 curl --cancel filename
1391 curl url
1392
1393 For internal use:
1394
1395 curl [--pak] [--forthismap] [--for filename filename...] url
1396         --pak: after downloading, load the package into the virtual file system
1397         --for filename...: only download of at least one of the named files is missing
1398         --forthismap: don't reconnect on failure
1399
1400 curl --clear_autodownload
1401         clears the download success/failure counters
1402
1403 curl --finish_autodownload
1404         if at least one download has been started, disconnect and drop to the menu
1405         once the last download completes successfully, reconnect to the current server
1406 ====================
1407 */
1408 static void Curl_Curl_f(cmd_state_t *cmd)
1409 {
1410         double maxspeed = 0;
1411         int i;
1412         int end;
1413         int loadtype = LOADTYPE_NONE;
1414         qbool forthismap = false;
1415         const char *url;
1416         const char *name = 0;
1417
1418         if(!curl_dll)
1419         {
1420                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1421                 return;
1422         }
1423
1424         if(!cl_curl_enabled.integer)
1425         {
1426                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1427                 return;
1428         }
1429
1430         if(Cmd_Argc(cmd) < 2)
1431         {
1432                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1433                 return;
1434         }
1435
1436         url = Cmd_Argv(cmd, Cmd_Argc(cmd) - 1);
1437         end = Cmd_Argc(cmd);
1438
1439         for(i = 1; i != end; ++i)
1440         {
1441                 const char *a = Cmd_Argv(cmd, i);
1442                 if(!strcmp(a, "--info"))
1443                 {
1444                         Curl_Info_f(cmd);
1445                         return;
1446                 }
1447                 else if(!strcmp(a, "--cancel"))
1448                 {
1449                         if(i == end - 1) // last argument
1450                                 Curl_CancelAll();
1451                         else
1452                         {
1453                                 downloadinfo *di = Curl_Find(url);
1454                                 if(di)
1455                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1456                                 else
1457                                         Con_Print("download not found\n");
1458                         }
1459                         return;
1460                 }
1461                 else if(!strcmp(a, "--pak"))
1462                 {
1463                         loadtype = LOADTYPE_PAK;
1464                 }
1465                 else if(!strcmp(a, "--cachepic"))
1466                 {
1467                         loadtype = LOADTYPE_CACHEPIC;
1468                 }
1469                 else if(!strcmp(a, "--skinframe"))
1470                 {
1471                         loadtype = LOADTYPE_SKINFRAME;
1472                 }
1473                 else if(!strcmp(a, "--for")) // must be last option
1474                 {
1475                         for(i = i + 1; i != end - 1; ++i)
1476                         {
1477                                 if(!FS_FileExists(Cmd_Argv(cmd, i)))
1478                                         goto needthefile; // why can't I have a "double break"?
1479                         }
1480                         // if we get here, we have all the files...
1481                         return;
1482                 }
1483                 else if(!strcmp(a, "--forthismap"))
1484                 {
1485                         forthismap = true;
1486                 }
1487                 else if(!strcmp(a, "--as"))
1488                 {
1489                         if(i < end - 1)
1490                         {
1491                                 ++i;
1492                                 name = Cmd_Argv(cmd, i);
1493                         }
1494                 }
1495                 else if(!strcmp(a, "--clear_autodownload"))
1496                 {
1497                         // mark all running downloads as "not for this map", so if they
1498                         // fail, it does not matter
1499                         Curl_Clear_forthismap();
1500                         return;
1501                 }
1502                 else if(!strcmp(a, "--finish_autodownload"))
1503                 {
1504                         if(numdownloads_added)
1505                         {
1506                                 char donecommand[256];
1507                                 if(cls.netcon)
1508                                 {
1509                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1510                                         {
1511                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1512                                                 Curl_CommandWhenDone(donecommand);
1513                                                 noclear = true;
1514                                                 CL_Disconnect();
1515                                                 noclear = false;
1516                                                 Curl_CheckCommandWhenDone();
1517                                         }
1518                                         else
1519                                                 Curl_Register_predownload();
1520                                 }
1521                         }
1522                         return;
1523                 }
1524                 else if(!strncmp(a, "--maxspeed=", 11))
1525                 {
1526                         maxspeed = atof(a + 11);
1527                 }
1528                 else if(*a == '-')
1529                 {
1530                         Con_Printf("curl: invalid option %s\n", a);
1531                         // but we ignore the option
1532                 }
1533         }
1534
1535 needthefile:
1536         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1537 }
1538
1539 /*
1540 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1541 {
1542         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1543         Z_Free(buffer);
1544 }
1545
1546 void Curl_CurlCat_f(cmd_state_t *cmd)
1547 {
1548         unsigned char *buf;
1549         const char *url = Cmd_Argv(cmd, 1);
1550         buf = Z_Malloc(16384);
1551         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1552 }
1553 */
1554
1555 /*
1556 ====================
1557 Curl_Init_Commands
1558
1559 loads the commands and cvars this library uses
1560 ====================
1561 */
1562 void Curl_Init_Commands(void)
1563 {
1564         Cvar_RegisterVariable (&cl_curl_enabled);
1565         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1566         Cvar_RegisterVariable (&cl_curl_maxspeed);
1567         Cvar_RegisterVariable (&sv_curl_defaulturl);
1568         Cvar_RegisterVariable (&sv_curl_serverpackages);
1569         Cvar_RegisterVariable (&sv_curl_maxspeed);
1570         Cvar_RegisterVariable (&cl_curl_useragent);
1571         Cvar_RegisterVariable (&cl_curl_useragent_append);
1572         Cvar_RegisterVariable (&developer_curl);
1573         Cmd_AddCommand(CF_CLIENT | CF_CLIENT_FROM_SERVER, "curl", Curl_Curl_f, "download data from an URL and add to search path");
1574         //Cmd_AddCommand(cmd_local, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1575 }
1576
1577 /*
1578 ====================
1579 Curl_GetDownloadInfo
1580
1581 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1582 The number of elements in the array is returned in int *nDownloads.
1583 const char **additional_info may be set to a string of additional user
1584 information, or to NULL if no such display shall occur. The returned
1585 array must be freed later using Z_Free.
1586 ====================
1587 */
1588 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1589 {
1590         int i;
1591         downloadinfo *di;
1592         Curl_downloadinfo_t *downinfo;
1593
1594         if(!curl_dll)
1595         {
1596                 *nDownloads = 0;
1597                 if(additional_info)
1598                         *additional_info = NULL;
1599                 return NULL;
1600         }
1601
1602         if (curl_mutex) Thread_LockMutex(curl_mutex);
1603
1604         i = 0;
1605         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1606                 ++i;
1607
1608         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1609         i = 0;
1610         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1611         {
1612                 // do not show infobars for background downloads
1613                 if(developer.integer <= 0)
1614                         if(di->buffer)
1615                                 continue;
1616                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1617                 if(di->curle)
1618                 {
1619                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1620                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1621                         downinfo[i].queued = false;
1622                 }
1623                 else
1624                 {
1625                         downinfo[i].queued = true;
1626                 }
1627                 ++i;
1628         }
1629
1630         if(additional_info)
1631         {
1632                 // TODO: can I clear command_when_done as soon as the first download fails?
1633                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1634                 {
1635                         if(!strncmp(command_when_done, "connect ", 8))
1636                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1637                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1638                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1639                         else
1640                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1641                         *additional_info = addinfo;
1642                 }
1643                 else
1644                         *additional_info = NULL;
1645         }
1646
1647         *nDownloads = i;
1648         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1649         return downinfo;
1650 }
1651
1652
1653 /*
1654 ====================
1655 Curl_FindPackURL
1656
1657 finds the URL where to find a given package.
1658
1659 For this, it reads a file "curl_urls.txt" of the following format:
1660
1661         data*.pk3       -
1662         revdm*.pk3      http://revdm/downloads/are/here/
1663         *                       http://any/other/stuff/is/here/
1664
1665 The URLs should end in /. If not, downloads will still work, but the cached files
1666 can't be just put into the data directory with the same download configuration
1667 (you might want to do this if you want to tag downloaded files from your
1668 server, but you should not). "-" means "don't download".
1669
1670 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1671 location instead.
1672
1673 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1674 this file for obvious reasons.
1675 ====================
1676 */
1677 static const char *Curl_FindPackURL(const char *filename)
1678 {
1679         static char foundurl[1024]; // invoked only by server
1680         fs_offset_t filesize;
1681         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1682         if(buf && filesize)
1683         {
1684                 // read lines of format "pattern url"
1685                 char *p = buf;
1686                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1687                 qbool eof = false;
1688
1689                 pattern = p;
1690                 while(!eof)
1691                 {
1692                         switch(*p)
1693                         {
1694                                 case 0:
1695                                         eof = true;
1696                                         // fallthrough
1697                                 case '\n':
1698                                 case '\r':
1699                                         if(pattern && url && patternend)
1700                                         {
1701                                                 if(!urlend)
1702                                                         urlend = p;
1703                                                 *patternend = 0;
1704                                                 *urlend = 0;
1705                                                 if(matchpattern(filename, pattern, true))
1706                                                 {
1707                                                         strlcpy(foundurl, url, sizeof(foundurl));
1708                                                         Z_Free(buf);
1709                                                         return foundurl;
1710                                                 }
1711                                         }
1712                                         pattern = NULL;
1713                                         patternend = NULL;
1714                                         url = NULL;
1715                                         urlend = NULL;
1716                                         break;
1717                                 case ' ':
1718                                 case '\t':
1719                                         if(pattern && !patternend)
1720                                                 patternend = p;
1721                                         else if(url && !urlend)
1722                                                 urlend = p;
1723                                         break;
1724                                 default:
1725                                         if(!pattern)
1726                                                 pattern = p;
1727                                         else if(pattern && patternend && !url)
1728                                                 url = p;
1729                                         break;
1730                         }
1731                         ++p;
1732                 }
1733         }
1734         if(buf)
1735                 Z_Free(buf);
1736         return sv_curl_defaulturl.string;
1737 }
1738
1739 typedef struct requirement_s
1740 {
1741         struct requirement_s *next;
1742         char filename[MAX_OSPATH];
1743 }
1744 requirement;
1745 static requirement *requirements = NULL;
1746
1747
1748 /*
1749 ====================
1750 Curl_RequireFile
1751
1752 Adds the given file to the list of requirements.
1753 ====================
1754 */
1755 void Curl_RequireFile(const char *filename)
1756 {
1757         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1758         req->next = requirements;
1759         strlcpy(req->filename, filename, sizeof(req->filename));
1760         requirements = req;
1761 }
1762
1763 /*
1764 ====================
1765 Curl_ClearRequirements
1766
1767 Clears the list of required files for playing on the current map.
1768 This should be called at every map change.
1769 ====================
1770 */
1771 void Curl_ClearRequirements(void)
1772 {
1773         while(requirements)
1774         {
1775                 requirement *req = requirements;
1776                 requirements = requirements->next;
1777                 Z_Free(req);
1778         }
1779 }
1780
1781 /*
1782 ====================
1783 Curl_SendRequirements
1784
1785 Makes the current host_clients download all files he needs.
1786 This is done by sending him the following console commands:
1787
1788         curl --clear_autodownload
1789         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1790         curl --finish_autodownload
1791 ====================
1792 */
1793 static qbool Curl_SendRequirement(const char *filename, qbool foundone, char *sendbuffer, size_t sendbuffer_len)
1794 {
1795         const char *p;
1796         const char *thispack = FS_WhichPack(filename);
1797         const char *packurl;
1798
1799         if(!thispack || !*thispack)
1800                 return false;
1801
1802         p = strrchr(thispack, '/');
1803         if(p)
1804                 thispack = p + 1;
1805
1806         packurl = Curl_FindPackURL(thispack);
1807
1808         if(packurl && *packurl && strcmp(packurl, "-"))
1809         {
1810                 if(!foundone)
1811                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1812
1813                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1814                 strlcat(sendbuffer, thispack, sendbuffer_len);
1815                 if(sv_curl_maxspeed.value > 0)
1816                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1817                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1818                 strlcat(sendbuffer, filename, sendbuffer_len);
1819                 strlcat(sendbuffer, " ", sendbuffer_len);
1820                 strlcat(sendbuffer, packurl, sendbuffer_len);
1821                 strlcat(sendbuffer, thispack, sendbuffer_len);
1822                 strlcat(sendbuffer, "\n", sendbuffer_len);
1823
1824                 return true;
1825         }
1826
1827         return false;
1828 }
1829 void Curl_SendRequirements(void)
1830 {
1831         // for each requirement, find the pack name
1832         char sendbuffer[4096] = "";
1833         requirement *req;
1834         qbool foundone = false;
1835         const char *p;
1836
1837         for(req = requirements; req; req = req->next)
1838                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1839
1840         p = sv_curl_serverpackages.string;
1841         while(COM_ParseToken_Simple(&p, false, false, true))
1842                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1843
1844         if(foundone)
1845                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1846
1847         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1848                 SV_ClientCommands("%s", sendbuffer);
1849         else
1850                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1851 }