libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5
   6 #include "image.h"
   7 #include "jpeg.h"
   8 #include "image_png.h"
   9
  10 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
  11 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
  12 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
  13 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  14 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  15 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  16 static cvar_t cl_curl_useragent = {0, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  17 static cvar_t cl_curl_useragent_append = {0, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  18
  19 /*
  20 =================================================================
  21
  22   Minimal set of definitions from libcurl
  23
  24   WARNING: for a matter of simplicity, several pointer types are
  25   casted to "void*", and most enumerated values are not included
  26
  27 =================================================================
  28 */
  29
  30 typedef struct CURL_s CURL;
  31 typedef struct CURLM_s CURLM;
  32 typedef struct curl_slist curl_slist;
  33 typedef enum
  34 {
  35         CURLE_OK = 0
  36 }
  37 CURLcode;
  38 typedef enum
  39 {
  40         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  41         CURLM_OK = 0
  42 }
  43 CURLMcode;
  44 #define CURL_GLOBAL_NOTHING 0
  45 #define CURL_GLOBAL_SSL 1
  46 #define CURL_GLOBAL_WIN32 2
  47 #define CURLOPTTYPE_LONG          0
  48 #define CURLOPTTYPE_OBJECTPOINT   10000
  49 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  50 #define CURLOPTTYPE_OFF_T         30000
  51 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  52 typedef enum
  53 {
  54         CINIT(WRITEDATA, OBJECTPOINT, 1),
  55         CINIT(URL,  OBJECTPOINT, 2),
  56         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  57         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  58         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  59         CINIT(REFERER, OBJECTPOINT, 16),
  60         CINIT(USERAGENT, OBJECTPOINT, 18),
  61         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  62         CINIT(LOW_SPEED_TIME, LONG, 20),
  63         CINIT(RESUME_FROM, LONG, 21),
  64         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  65         CINIT(POST, LONG, 47),         /* HTTP POST method */
  66         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  67         CINIT(POSTFIELDSIZE, LONG, 60),
  68         CINIT(PRIVATE, OBJECTPOINT, 103),
  69         CINIT(PROTOCOLS, LONG, 181),
  70         CINIT(REDIR_PROTOCOLS, LONG, 182)
  71 }
  72 CURLoption;
  73 #define CURLPROTO_HTTP   (1<<0)
  74 #define CURLPROTO_HTTPS  (1<<1)
  75 #define CURLPROTO_FTP    (1<<2)
  76 typedef enum
  77 {
  78         CURLINFO_TEXT = 0,
  79         CURLINFO_HEADER_IN,    /* 1 */
  80         CURLINFO_HEADER_OUT,   /* 2 */
  81         CURLINFO_DATA_IN,      /* 3 */
  82         CURLINFO_DATA_OUT,     /* 4 */
  83         CURLINFO_SSL_DATA_IN,  /* 5 */
  84         CURLINFO_SSL_DATA_OUT, /* 6 */
  85         CURLINFO_END
  86 }
  87 curl_infotype;
  88 #define CURLINFO_STRING   0x100000
  89 #define CURLINFO_LONG     0x200000
  90 #define CURLINFO_DOUBLE   0x300000
  91 #define CURLINFO_SLIST    0x400000
  92 #define CURLINFO_MASK     0x0fffff
  93 #define CURLINFO_TYPEMASK 0xf00000
  94 typedef enum
  95 {
  96         CURLINFO_NONE, /* first, never use this */
  97         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  98         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  99         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
 100         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
 101         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
 102         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
 103         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 104         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 105         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 106         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 107         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 108         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 109         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 110         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 111         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 112         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 113         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 114         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 115         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 116         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 117         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 118         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 119         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 120         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 121         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 122         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 123         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 124 }
 125 CURLINFO;
 126
 127 typedef enum
 128 {
 129         CURLMSG_NONE, /* first, not used */
 130         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 131                                          the CURLcode of the transfer */
 132         CURLMSG_LAST
 133 }
 134 CURLMSG;
 135 typedef struct
 136 {
 137         CURLMSG msg;       /* what this message means */
 138         CURL *easy_handle; /* the handle it concerns */
 139         union
 140         {
 141                 void *whatever;    /* message-specific data */
 142                 CURLcode result;   /* return code for transfer */
 143         }
 144         data;
 145 }
 146 CURLMsg;
 147
 148 static void (*qcurl_global_init) (long flags);
 149 static void (*qcurl_global_cleanup) (void);
 150
 151 static CURL * (*qcurl_easy_init) (void);
 152 static void (*qcurl_easy_cleanup) (CURL *handle);
 153 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 154 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 155 static const char * (*qcurl_easy_strerror) (CURLcode);
 156
 157 static CURLM * (*qcurl_multi_init) (void);
 158 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 159 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 160 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 161 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 162 static void (*qcurl_multi_cleanup) (CURLM *);
 163 static const char * (*qcurl_multi_strerror) (CURLcode);
 164 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 165 static void (*qcurl_slist_free_all) (curl_slist *list);
 166
 167 static dllfunction_t curlfuncs[] =
 168 {
 169         {"curl_global_init",            (void **) &qcurl_global_init},
 170         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 171         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 172         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 173         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 174         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 175         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 176         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 177         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 178         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 179         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 180         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 181         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 182         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 183         {"curl_slist_append",           (void **) &qcurl_slist_append},
 184         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 185         {NULL, NULL}
 186 };
 187
 188 // Handle for CURL DLL
 189 static dllhandle_t curl_dll = NULL;
 190 // will be checked at many places to find out if qcurl calls are allowed
 191
 192 #define LOADTYPE_NONE 0
 193 #define LOADTYPE_PAK 1
 194 #define LOADTYPE_CACHEPIC 2
 195 #define LOADTYPE_SKINFRAME 3
 196
 197 void *curl_mutex = NULL;
 198
 199 typedef struct downloadinfo_s
 200 {
 201         char filename[MAX_OSPATH];
 202         char url[1024];
 203         char referer[256];
 204         qfile_t *stream;
 205         fs_offset_t startpos;
 206         CURL *curle;
 207         qboolean started;
 208         int loadtype;
 209         unsigned long bytes_received; // for buffer
 210         double bytes_received_curl; // for throttling
 211         double bytes_sent_curl; // for throttling
 212         struct downloadinfo_s *next, *prev;
 213         qboolean forthismap;
 214         double maxspeed;
 215         curl_slist *slist; // http headers
 216
 217         unsigned char *buffer;
 218         size_t buffersize;
 219         curl_callback_t callback;
 220         void *callback_data;
 221
 222         const unsigned char *postbuf;
 223         size_t postbufsize;
 224         const char *post_content_type;
 225         const char *extraheaders;
 226 }
 227 downloadinfo;
 228 static downloadinfo *downloads = NULL;
 229 static int numdownloads = 0;
 230
 231 static qboolean noclear = FALSE;
 232
 233 static int numdownloads_fail = 0;
 234 static int numdownloads_success = 0;
 235 static int numdownloads_added = 0;
 236 static char command_when_done[256] = "";
 237 static char command_when_error[256] = "";
 238
 239 /*
 240 ====================
 241 Curl_CommandWhenDone
 242
 243 Sets the command which is to be executed when the last download completes AND
 244 all downloads since last server connect ended with a successful status.
 245 Setting the command to NULL clears it.
 246 ====================
 247 */
 248 static void Curl_CommandWhenDone(const char *cmd)
 249 {
 250         if(!curl_dll)
 251                 return;
 252         if(cmd)
 253                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 254         else
 255                 *command_when_done = 0;
 256 }
 257
 258 /*
 259 FIXME
 260 Do not use yet. Not complete.
 261 Problem: what counts as an error?
 262 */
 263
 264 static void Curl_CommandWhenError(const char *cmd)
 265 {
 266         if(!curl_dll)
 267                 return;
 268         if(cmd)
 269                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 270         else
 271                 *command_when_error = 0;
 272 }
 273
 274 /*
 275 ====================
 276 Curl_Clear_forthismap
 277
 278 Clears the "will disconnect on failure" flags.
 279 ====================
 280 */
 281 void Curl_Clear_forthismap(void)
 282 {
 283         downloadinfo *di;
 284         if(noclear)
 285                 return;
 286         if (curl_mutex) Thread_LockMutex(curl_mutex);
 287         for(di = downloads; di; di = di->next)
 288                 di->forthismap = false;
 289         Curl_CommandWhenError(NULL);
 290         Curl_CommandWhenDone(NULL);
 291         numdownloads_fail = 0;
 292         numdownloads_success = 0;
 293         numdownloads_added = 0;
 294         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 295 }
 296
 297 /*
 298 ====================
 299 Curl_Have_forthismap
 300
 301 Returns true if a download needed for the current game is running.
 302 ====================
 303 */
 304 qboolean Curl_Have_forthismap(void)
 305 {
 306         return numdownloads_added != 0;
 307 }
 308
 309 void Curl_Register_predownload(void)
 310 {
 311         if (curl_mutex) Thread_LockMutex(curl_mutex);
 312         Curl_CommandWhenDone("cl_begindownloads");
 313         Curl_CommandWhenError("cl_begindownloads");
 314         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 315 }
 316
 317 /*
 318 ====================
 319 Curl_CheckCommandWhenDone
 320
 321 Checks if a "done command" is to be executed.
 322 All downloads finished, at least one success since connect, no single failure
 323 -> execute the command.
 324 */
 325 static void Curl_CheckCommandWhenDone(void)
 326 {
 327         if(!curl_dll)
 328                 return;
 329         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 330         {
 331                 if(numdownloads_fail == 0)
 332                 {
 333                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 334                         Cbuf_AddText("\n");
 335                         Cbuf_AddText(command_when_done);
 336                         Cbuf_AddText("\n");
 337                 }
 338                 else
 339                 {
 340                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 341                         Cbuf_AddText("\n");
 342                         Cbuf_AddText(command_when_error);
 343                         Cbuf_AddText("\n");
 344                 }
 345                 Curl_Clear_forthismap();
 346         }
 347 }
 348
 349 /*
 350 ====================
 351 CURL_CloseLibrary
 352
 353 Load the cURL DLL
 354 ====================
 355 */
 356 static qboolean CURL_OpenLibrary (void)
 357 {
 358         const char* dllnames [] =
 359         {
 360 #if defined(WIN32)
 361                 "libcurl-4.dll",
 362                 "libcurl-3.dll",
 363 #elif defined(MACOSX)
 364                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 365                 "libcurl.3.dylib", // Mac OS X Tiger
 366                 "libcurl.2.dylib", // Mac OS X Panther
 367 #else
 368                 "libcurl.so.4",
 369                 "libcurl.so.3",
 370                 "libcurl.so", // FreeBSD
 371 #endif
 372                 NULL
 373         };
 374
 375         // Already loaded?
 376         if (curl_dll)
 377                 return true;
 378
 379         // Load the DLL
 380         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
 381 }
 382
 383
 384 /*
 385 ====================
 386 CURL_CloseLibrary
 387
 388 Unload the cURL DLL
 389 ====================
 390 */
 391 static void CURL_CloseLibrary (void)
 392 {
 393         Sys_UnloadLibrary (&curl_dll);
 394 }
 395
 396
 397 static CURLM *curlm = NULL;
 398 static double bytes_received = 0; // used for bandwidth throttling
 399 static double bytes_sent = 0; // used for bandwidth throttling
 400 static double curltime = 0;
 401
 402 /*
 403 ====================
 404 CURL_fwrite
 405
 406 fwrite-compatible function that writes the data to a file. libcurl can call
 407 this.
 408 ====================
 409 */
 410 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 411 {
 412         fs_offset_t ret = -1;
 413         size_t bytes = size * nmemb;
 414         downloadinfo *di = (downloadinfo *) vdi;
 415
 416         if(di->buffer)
 417         {
 418                 if(di->bytes_received + bytes <= di->buffersize)
 419                 {
 420                         memcpy(di->buffer + di->bytes_received, data, bytes);
 421                         ret = bytes;
 422                 }
 423                 // otherwise: buffer overrun, ret stays -1
 424         }
 425
 426         if(di->stream)
 427         {
 428                 ret = FS_Write(di->stream, data, bytes);
 429         }
 430
 431         di->bytes_received += bytes;
 432
 433         return ret; // why not ret / nmemb?
 434 }
 435
 436 typedef enum
 437 {
 438         CURL_DOWNLOAD_SUCCESS = 0,
 439         CURL_DOWNLOAD_FAILED,
 440         CURL_DOWNLOAD_ABORTED,
 441         CURL_DOWNLOAD_SERVERERROR
 442 }
 443 CurlStatus;
 444
 445 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 446 {
 447         downloadinfo *di = (downloadinfo *) cbdata;
 448         switch(status)
 449         {
 450                 case CURLCBSTATUS_OK:
 451                         Con_DPrintf("Download of %s: OK\n", di->filename);
 452                         break;
 453                 case CURLCBSTATUS_FAILED:
 454                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 455                         break;
 456                 case CURLCBSTATUS_ABORTED:
 457                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 458                         break;
 459                 case CURLCBSTATUS_SERVERERROR:
 460                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 461                         break;
 462                 case CURLCBSTATUS_UNKNOWN:
 463                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 464                         break;
 465                 default:
 466                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 467                         break;
 468         }
 469 }
 470
 471 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 472 {
 473         curl_default_callback(status, length_received, buffer, cbdata);
 474 }
 475
 476 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
 477 {
 478         unsigned char *pixels = NULL;
 479         fs_offset_t filesize = 0;
 480         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
 481         if(data)
 482         {
 483                 int mip = 0;
 484                 if(!strcmp(content_type, "image/jpeg"))
 485                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 486                 else if(!strcmp(content_type, "image/png"))
 487                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 488                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
 489                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 490                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
 491                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 492                 else
 493                         Con_Printf("Did not detect content type: %s\n", content_type);
 494                 Mem_Free(data);
 495         }
 496         // do we call Image_MakeLinearColorsFromsRGB or not?
 497         return pixels;
 498 }
 499
 500 /*
 501 ====================
 502 Curl_EndDownload
 503
 504 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 505 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 506 code from libcurl, or 0, if another error has occurred.
 507 ====================
 508 */
 509 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 510 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
 511 {
 512         char content_type[64];
 513         qboolean ok = false;
 514         if(!curl_dll)
 515                 return;
 516         switch(status)
 517         {
 518                 case CURL_DOWNLOAD_SUCCESS:
 519                         ok = true;
 520                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 521                         break;
 522                 case CURL_DOWNLOAD_FAILED:
 523                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 524                         break;
 525                 case CURL_DOWNLOAD_ABORTED:
 526                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 527                         break;
 528                 case CURL_DOWNLOAD_SERVERERROR:
 529                         // reopen to enforce it to have zero bytes again
 530                         if(di->stream)
 531                         {
 532                                 FS_Close(di->stream);
 533                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 534                         }
 535
 536                         if(di->callback)
 537                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 538                         break;
 539                 default:
 540                         if(di->callback)
 541                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 542                         break;
 543         }
 544         if(content_type_)
 545                 strlcpy(content_type, content_type_, sizeof(content_type));
 546         else
 547                 *content_type = 0;
 548
 549         if(di->curle)
 550         {
 551                 qcurl_multi_remove_handle(curlm, di->curle);
 552                 qcurl_easy_cleanup(di->curle);
 553                 if(di->slist)
 554                         qcurl_slist_free_all(di->slist);
 555         }
 556
 557         if(!di->callback && ok && !di->bytes_received)
 558         {
 559                 Con_Printf("ERROR: empty file\n");
 560                 ok = false;
 561         }
 562
 563         if(di->stream)
 564                 FS_Close(di->stream);
 565
 566 #define CLEAR_AND_RETRY() \
 567         do \
 568         { \
 569                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
 570                 FS_Close(di->stream); \
 571                 if(di->startpos && !di->callback) \
 572                 { \
 573                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
 574                         di->forthismap = false; \
 575                 } \
 576         } \
 577         while(0)
 578
 579         if(ok && di->loadtype == LOADTYPE_PAK)
 580         {
 581                 ok = FS_AddPack(di->filename, NULL, true);
 582                 if(!ok)
 583                         CLEAR_AND_RETRY();
 584         }
 585         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
 586         {
 587                 const char *p;
 588                 unsigned char *pixels = NULL;
 589
 590                 p = di->filename;
 591 #ifdef WE_ARE_EVIL
 592                 if(!strncmp(p, "dlcache/", 8))
 593                         p += 8;
 594 #endif
 595
 596                 pixels = decode_image(di, content_type);
 597                 if(pixels)
 598                         Draw_NewPic(p, image_width, image_height, true, pixels);
 599                 else
 600                         CLEAR_AND_RETRY();
 601         }
 602         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
 603         {
 604                 const char *p;
 605                 unsigned char *pixels = NULL;
 606
 607                 p = di->filename;
 608 #ifdef WE_ARE_EVIL
 609                 if(!strncmp(p, "dlcache/", 8))
 610                         p += 8;
 611 #endif
 612
 613                 pixels = decode_image(di, content_type);
 614                 if(pixels)
 615                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, false); // TODO what sRGB argument to put here?
 616                 else
 617                         CLEAR_AND_RETRY();
 618         }
 619
 620         if(di->prev)
 621                 di->prev->next = di->next;
 622         else
 623                 downloads = di->next;
 624         if(di->next)
 625                 di->next->prev = di->prev;
 626
 627         --numdownloads;
 628         if(di->forthismap)
 629         {
 630                 if(ok)
 631                         ++numdownloads_success;
 632                 else
 633                         ++numdownloads_fail;
 634         }
 635         Z_Free(di);
 636 }
 637
 638 /*
 639 ====================
 640 CleanURL
 641
 642 Returns a "cleaned up" URL for display (to strip login data)
 643 ====================
 644 */
 645 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 646 {
 647         const char *p, *q, *r;
 648
 649         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 650         p = strstr(url, "://");
 651         if(p)
 652         {
 653                 q = strchr(p + 3, '@');
 654                 if(q)
 655                 {
 656                         r = strchr(p + 3, '/');
 657                         if(!r || q < r)
 658                         {
 659                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 660                                 return urlbuf;
 661                         }
 662                 }
 663         }
 664
 665         return url;
 666 }
 667
 668 /*
 669 ====================
 670 CheckPendingDownloads
 671
 672 checks if there are free download slots to start new downloads in.
 673 To not start too many downloads at once, only one download is added at a time,
 674 up to a maximum number of cl_curl_maxdownloads are running.
 675 ====================
 676 */
 677 static void CheckPendingDownloads(void)
 678 {
 679         const char *h;
 680         char urlbuf[1024];
 681         char vabuf[1024];
 682         if(!curl_dll)
 683                 return;
 684         if(numdownloads < cl_curl_maxdownloads.integer)
 685         {
 686                 downloadinfo *di;
 687                 for(di = downloads; di; di = di->next)
 688                 {
 689                         if(!di->started)
 690                         {
 691                                 if(!di->buffer)
 692                                 {
 693                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 694
 695                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 696                                         if(!di->stream)
 697                                         {
 698                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 699                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
 700                                                 return;
 701                                         }
 702                                         FS_Seek(di->stream, 0, SEEK_END);
 703                                         di->startpos = FS_Tell(di->stream);
 704
 705                                         if(di->startpos > 0)
 706                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 707                                         Con_Print("...\n");
 708                                 }
 709                                 else
 710                                 {
 711                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 712                                         di->startpos = 0;
 713                                 }
 714
 715                                 di->curle = qcurl_easy_init();
 716                                 di->slist = NULL;
 717                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 718                                 if(cl_curl_useragent.integer)
 719                                 {
 720                                         const char *ua
 721 #ifdef HTTP_USER_AGENT
 722                                                 = HTTP_USER_AGENT;
 723 #else
 724                                                 = engineversion;
 725 #endif
 726                                         if(!ua)
 727                                                 ua = "";
 728                                         if(*cl_curl_useragent_append.string)
 729                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 730                                                         ua,
 731                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 732                                                                 ? " "
 733                                                                 : "",
 734                                                         cl_curl_useragent_append.string);
 735                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 736                                 }
 737                                 else
 738                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 739                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 740                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 741                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 742                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 743                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 744                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 745                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 746                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 747                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 748                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 749                                 {
 750                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 751                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 752                                 }
 753                                 if(di->post_content_type)
 754                                 {
 755                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 756                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 757                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 758                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 759                                 }
 760
 761                                 // parse extra headers into slist
 762                                 // \n separated list!
 763                                 h = di->extraheaders;
 764                                 while(h)
 765                                 {
 766                                         const char *hh = strchr(h, '\n');
 767                                         if(hh)
 768                                         {
 769                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 770                                                 memcpy(buf, h, hh - h);
 771                                                 buf[hh - h] = 0;
 772                                                 di->slist = qcurl_slist_append(di->slist, buf);
 773                                                 h = hh + 1;
 774                                         }
 775                                         else
 776                                         {
 777                                                 di->slist = qcurl_slist_append(di->slist, h);
 778                                                 h = NULL;
 779                                         }
 780                                 }
 781
 782                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 783
 784                                 qcurl_multi_add_handle(curlm, di->curle);
 785                                 di->started = true;
 786                                 ++numdownloads;
 787                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 788                                         break;
 789                         }
 790                 }
 791         }
 792 }
 793
 794 /*
 795 ====================
 796 Curl_Init
 797
 798 this function MUST be called before using anything else in this file.
 799 On Win32, this must be called AFTER WSAStartup has been done!
 800 ====================
 801 */
 802 void Curl_Init(void)
 803 {
 804         CURL_OpenLibrary();
 805         if(!curl_dll)
 806                 return;
 807         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 808         qcurl_global_init(CURL_GLOBAL_NOTHING);
 809         curlm = qcurl_multi_init();
 810 }
 811
 812 /*
 813 ====================
 814 Curl_Shutdown
 815
 816 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 817 ====================
 818 */
 819 void Curl_ClearRequirements(void);
 820 void Curl_Shutdown(void)
 821 {
 822         if(!curl_dll)
 823                 return;
 824         Curl_ClearRequirements();
 825         Curl_CancelAll();
 826         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 827         CURL_CloseLibrary();
 828         curl_dll = NULL;
 829 }
 830
 831 /*
 832 ====================
 833 Curl_Find
 834
 835 Finds the internal information block for a download given by file name.
 836 ====================
 837 */
 838 static downloadinfo *Curl_Find(const char *filename)
 839 {
 840         downloadinfo *di;
 841         if(!curl_dll)
 842                 return NULL;
 843         for(di = downloads; di; di = di->next)
 844                 if(!strcasecmp(di->filename, filename))
 845                         return di;
 846         return NULL;
 847 }
 848
 849 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 850 {
 851         downloadinfo *di;
 852         if(!curl_dll)
 853                 return;
 854         for(di = downloads; di; )
 855         {
 856                 if(di->callback == callback && di->callback_data == cbdata)
 857                 {
 858                         di->callback = curl_quiet_callback; // do NOT call the callback
 859                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
 860                         di = downloads;
 861                 }
 862                 else
 863                         di = di->next;
 864         }
 865 }
 866
 867 /*
 868 ====================
 869 Curl_Begin
 870
 871 Starts a download of a given URL to the file name portion of this URL (or name
 872 if given) in the "dlcache/" folder.
 873 ====================
 874 */
 875 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 876 {
 877         if(buf)
 878                 if(loadtype != LOADTYPE_NONE)
 879                         Host_Error("Curl_Begin: loadtype and buffer are both set");
 880
 881         if(!curl_dll || !cl_curl_enabled.integer)
 882         {
 883                 return false;
 884         }
 885         else
 886         {
 887                 char fn[MAX_OSPATH];
 888                 char urlbuf[1024];
 889                 const char *p, *q;
 890                 size_t length;
 891                 downloadinfo *di;
 892
 893                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 894                 p = strchr(URL, ':');
 895                 if(p)
 896                 {
 897                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 898                         {
 899                                 char addressstring[128];
 900                                 *addressstring = 0;
 901                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 902                                 q = strchr(addressstring, ':');
 903                                 if(!q)
 904                                         q = addressstring + strlen(addressstring);
 905                                 if(*addressstring)
 906                                 {
 907                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 908                                         URL = urlbuf;
 909                                 }
 910                         }
 911                 }
 912
 913                 // Note: This extraction of the file name portion is NOT entirely correct.
 914                 //
 915                 // It does the following:
 916                 //
 917                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 918                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 919                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 920                 //
 921                 // However, I'd like to keep this "buggy" behavior so that PHP script
 922                 // authors can write download scripts without having to enable
 923                 // AcceptPathInfo on Apache. They just have to ensure that their script
 924                 // can be called with such a "fake" path name like
 925                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 926                 //
 927                 // By the way, such PHP scripts should either send the file or a
 928                 // "Location:" redirect; PHP code example:
 929                 //
 930                 //   header("Location: http://www.example.com/");
 931                 //
 932                 // By the way, this will set User-Agent to something like
 933                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 934                 // dp://serverhost:serverport/ so you can filter on this; an example
 935                 // httpd log file line might be:
 936                 //
 937                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 938
 939                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 940
 941                 if(buf)
 942                 {
 943                         if(!name)
 944                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 945                 }
 946                 else
 947                 {
 948                         if(!name)
 949                         {
 950                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 951                                 p = strrchr(name, '/');
 952                                 p = p ? (p+1) : name;
 953                                 q = strchr(p, '?');
 954                                 length = q ? (size_t)(q - p) : strlen(p);
 955                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 956                         }
 957                         else
 958                         {
 959                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
 960                         }
 961
 962                         name = fn; // make it point back
 963
 964                         // already downloading the file?
 965                         {
 966                                 downloadinfo *di = Curl_Find(fn);
 967                                 if(di)
 968                                 {
 969                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 970
 971                                         // however, if it was not for this map yet...
 972                                         if(forthismap && !di->forthismap)
 973                                         {
 974                                                 di->forthismap = true;
 975                                                 // this "fakes" a download attempt so the client will wait for
 976                                                 // the download to finish and then reconnect
 977                                                 ++numdownloads_added;
 978                                         }
 979
 980                                         return false;
 981                                 }
 982                         }
 983
 984                         if(FS_FileExists(fn))
 985                         {
 986                                 if(loadtype == LOADTYPE_PAK)
 987                                 {
 988                                         qboolean already_loaded;
 989                                         if(FS_AddPack(fn, &already_loaded, true))
 990                                         {
 991                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 992                                                 if(already_loaded)
 993                                                         Con_DPrintf("(pak was already loaded)\n");
 994                                                 else
 995                                                 {
 996                                                         if(forthismap)
 997                                                         {
 998                                                                 ++numdownloads_added;
 999                                                                 ++numdownloads_success;
1000                                                         }
1001                                                 }
1002
1003                                                 return false;
1004                                         }
1005                                         else
1006                                         {
1007                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1008                                                 if(f)
1009                                                 {
1010                                                         char buf[4] = {0};
1011                                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
1012
1013                                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
1014                                                         {
1015                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1016                                                                 FS_Close(f);
1017                                                                 f = FS_OpenRealFile(fn, "wb", false);
1018                                                                 if(f)
1019                                                                         FS_Close(f);
1020                                                         }
1021                                                         else
1022                                                         {
1023                                                                 // OK
1024                                                                 FS_Close(f);
1025                                                         }
1026                                                 }
1027                                         }
1028                                 }
1029                                 else
1030                                 {
1031                                         // never resume these
1032                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1033                                         if(f)
1034                                                 FS_Close(f);
1035                                 }
1036                         }
1037                 }
1038
1039                 // if we get here, we actually want to download... so first verify the
1040                 // URL scheme (so one can't read local files using file://)
1041                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1042                 {
1043                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1044                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1045                         return false;
1046                 }
1047
1048                 if(forthismap)
1049                         ++numdownloads_added;
1050                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1051                 strlcpy(di->filename, name, sizeof(di->filename));
1052                 strlcpy(di->url, URL, sizeof(di->url));
1053                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1054                 di->forthismap = forthismap;
1055                 di->stream = NULL;
1056                 di->startpos = 0;
1057                 di->curle = NULL;
1058                 di->started = false;
1059                 di->loadtype = loadtype;
1060                 di->maxspeed = maxspeed;
1061                 di->bytes_received = 0;
1062                 di->bytes_received_curl = 0;
1063                 di->bytes_sent_curl = 0;
1064                 di->extraheaders = extraheaders;
1065                 di->next = downloads;
1066                 di->prev = NULL;
1067                 if(di->next)
1068                         di->next->prev = di;
1069
1070                 di->buffer = buf;
1071                 di->buffersize = bufsize;
1072                 if(callback == NULL)
1073                 {
1074                         di->callback = curl_default_callback;
1075                         di->callback_data = di;
1076                 }
1077                 else
1078                 {
1079                         di->callback = callback;
1080                         di->callback_data = cbdata;
1081                 }
1082
1083                 if(post_content_type)
1084                 {
1085                         di->post_content_type = post_content_type;
1086                         di->postbuf = postbuf;
1087                         di->postbufsize = postbufsize;
1088                 }
1089                 else
1090                 {
1091                         di->post_content_type = NULL;
1092                         di->postbuf = NULL;
1093                         di->postbufsize = 0;
1094                 }
1095
1096                 downloads = di;
1097                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1098                 return true;
1099         }
1100 }
1101
1102 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qboolean forthismap)
1103 {
1104         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1105 }
1106 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1107 {
1108         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1109 }
1110 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1111 {
1112         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1113 }
1114
1115 /*
1116 ====================
1117 Curl_Run
1118
1119 call this regularily as this will always download as much as possible without
1120 blocking.
1121 ====================
1122 */
1123 void Curl_Run(void)
1124 {
1125         double maxspeed;
1126         downloadinfo *di;
1127
1128         noclear = FALSE;
1129
1130         if(!cl_curl_enabled.integer)
1131                 return;
1132
1133         if(!curl_dll)
1134                 return;
1135
1136         if (curl_mutex) Thread_LockMutex(curl_mutex);
1137
1138         Curl_CheckCommandWhenDone();
1139
1140         if(!downloads)
1141         {
1142                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1143                 return;
1144         }
1145
1146         if(realtime < curltime) // throttle
1147         {
1148                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1149                 return;
1150         }
1151
1152         {
1153                 int remaining;
1154                 CURLMcode mc;
1155
1156                 do
1157                 {
1158                         mc = qcurl_multi_perform(curlm, &remaining);
1159                 }
1160                 while(mc == CURLM_CALL_MULTI_PERFORM);
1161
1162                 for(di = downloads; di; di = di->next)
1163                 {
1164                         double b = 0;
1165                         if(di->curle)
1166                         {
1167                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1168                                 bytes_sent += (b - di->bytes_sent_curl);
1169                                 di->bytes_sent_curl = b;
1170                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1171                                 bytes_sent += (b - di->bytes_received_curl);
1172                                 di->bytes_received_curl = b;
1173                         }
1174                 }
1175
1176                 for(;;)
1177                 {
1178                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1179                         if(!msg)
1180                                 break;
1181                         if(msg->msg == CURLMSG_DONE)
1182                         {
1183                                 const char *ct = NULL;
1184                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1185                                 CURLcode result;
1186                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1187                                 result = msg->data.result;
1188                                 if(result)
1189                                 {
1190                                         failed = CURL_DOWNLOAD_FAILED;
1191                                 }
1192                                 else
1193                                 {
1194                                         long code;
1195                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1196                                         switch(code / 100)
1197                                         {
1198                                                 case 4: // e.g. 404?
1199                                                 case 5: // e.g. 500?
1200                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1201                                                         result = (CURLcode) code;
1202                                                         break;
1203                                         }
1204                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1205                                 }
1206
1207                                 Curl_EndDownload(di, failed, result, ct);
1208                         }
1209                 }
1210         }
1211
1212         CheckPendingDownloads();
1213
1214         // when will we curl the next time?
1215         // we will wait a bit to ensure our download rate is kept.
1216         // we now know that realtime >= curltime... so set up a new curltime
1217
1218         // use the slowest allowing download to derive the maxspeed... this CAN
1219         // be done better, but maybe later
1220         maxspeed = cl_curl_maxspeed.value;
1221         for(di = downloads; di; di = di->next)
1222                 if(di->maxspeed > 0)
1223                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1224                                 maxspeed = di->maxspeed;
1225
1226         if(maxspeed > 0)
1227         {
1228                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1229                 curltime = realtime + bytes / (maxspeed * 1024.0);
1230                 bytes_sent = 0;
1231                 bytes_received = 0;
1232         }
1233         else
1234                 curltime = realtime;
1235
1236         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1237 }
1238
1239 /*
1240 ====================
1241 Curl_CancelAll
1242
1243 Stops ALL downloads.
1244 ====================
1245 */
1246 void Curl_CancelAll(void)
1247 {
1248         if(!curl_dll)
1249                 return;
1250
1251         if (curl_mutex) Thread_LockMutex(curl_mutex);
1252
1253         while(downloads)
1254         {
1255                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1256                 // INVARIANT: downloads will point to the next download after that!
1257         }
1258
1259         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1260 }
1261
1262 /*
1263 ====================
1264 Curl_Running
1265
1266 returns true iff there is a download running.
1267 ====================
1268 */
1269 qboolean Curl_Running(void)
1270 {
1271         if(!curl_dll)
1272                 return false;
1273
1274         return downloads != NULL;
1275 }
1276
1277 /*
1278 ====================
1279 Curl_GetDownloadAmount
1280
1281 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1282 for the given download.
1283 ====================
1284 */
1285 static double Curl_GetDownloadAmount(downloadinfo *di)
1286 {
1287         if(!curl_dll)
1288                 return -2;
1289         if(di->curle)
1290         {
1291                 double length;
1292                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1293                 if(length > 0)
1294                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1295                 else
1296                         return 0;
1297         }
1298         else
1299                 return -1;
1300 }
1301
1302 /*
1303 ====================
1304 Curl_GetDownloadSpeed
1305
1306 returns the speed of the given download in bytes per second
1307 ====================
1308 */
1309 static double Curl_GetDownloadSpeed(downloadinfo *di)
1310 {
1311         if(!curl_dll)
1312                 return -2;
1313         if(di->curle)
1314         {
1315                 double speed;
1316                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1317                 return speed;
1318         }
1319         else
1320                 return -1;
1321 }
1322
1323 /*
1324 ====================
1325 Curl_Info_f
1326
1327 prints the download list
1328 ====================
1329 */
1330 // TODO rewrite using Curl_GetDownloadInfo?
1331 static void Curl_Info_f(void)
1332 {
1333         downloadinfo *di;
1334         char urlbuf[1024];
1335         if(!curl_dll)
1336                 return;
1337         if(Curl_Running())
1338         {
1339                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1340                 Con_Print("Currently running downloads:\n");
1341                 for(di = downloads; di; di = di->next)
1342                 {
1343                         double speed, percent;
1344                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1345                         percent = 100.0 * Curl_GetDownloadAmount(di);
1346                         speed = Curl_GetDownloadSpeed(di);
1347                         if(percent >= 0)
1348                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1349                         else
1350                                 Con_Print("(queued)\n");
1351                 }
1352                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1353         }
1354         else
1355         {
1356                 Con_Print("No downloads running.\n");
1357         }
1358 }
1359
1360 /*
1361 ====================
1362 Curl_Curl_f
1363
1364 implements the "curl" console command
1365
1366 curl --info
1367 curl --cancel
1368 curl --cancel filename
1369 curl url
1370
1371 For internal use:
1372
1373 curl [--pak] [--forthismap] [--for filename filename...] url
1374         --pak: after downloading, load the package into the virtual file system
1375         --for filename...: only download of at least one of the named files is missing
1376         --forthismap: don't reconnect on failure
1377
1378 curl --clear_autodownload
1379         clears the download success/failure counters
1380
1381 curl --finish_autodownload
1382         if at least one download has been started, disconnect and drop to the menu
1383         once the last download completes successfully, reconnect to the current server
1384 ====================
1385 */
1386 static void Curl_Curl_f(void)
1387 {
1388         double maxspeed = 0;
1389         int i;
1390         int end;
1391         int loadtype = LOADTYPE_NONE;
1392         qboolean forthismap = false;
1393         const char *url;
1394         const char *name = 0;
1395
1396         if(!curl_dll)
1397         {
1398                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1399                 return;
1400         }
1401
1402         if(!cl_curl_enabled.integer)
1403         {
1404                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1405                 return;
1406         }
1407
1408         if(Cmd_Argc() < 2)
1409         {
1410                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1411                 return;
1412         }
1413
1414         url = Cmd_Argv(Cmd_Argc() - 1);
1415         end = Cmd_Argc();
1416
1417         for(i = 1; i != end; ++i)
1418         {
1419                 const char *a = Cmd_Argv(i);
1420                 if(!strcmp(a, "--info"))
1421                 {
1422                         Curl_Info_f();
1423                         return;
1424                 }
1425                 else if(!strcmp(a, "--cancel"))
1426                 {
1427                         if(i == end - 1) // last argument
1428                                 Curl_CancelAll();
1429                         else
1430                         {
1431                                 downloadinfo *di = Curl_Find(url);
1432                                 if(di)
1433                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1434                                 else
1435                                         Con_Print("download not found\n");
1436                         }
1437                         return;
1438                 }
1439                 else if(!strcmp(a, "--pak"))
1440                 {
1441                         loadtype = LOADTYPE_PAK;
1442                 }
1443                 else if(!strcmp(a, "--cachepic"))
1444                 {
1445                         loadtype = LOADTYPE_CACHEPIC;
1446                 }
1447                 else if(!strcmp(a, "--skinframe"))
1448                 {
1449                         loadtype = LOADTYPE_SKINFRAME;
1450                 }
1451                 else if(!strcmp(a, "--for")) // must be last option
1452                 {
1453                         for(i = i + 1; i != end - 1; ++i)
1454                         {
1455                                 if(!FS_FileExists(Cmd_Argv(i)))
1456                                         goto needthefile; // why can't I have a "double break"?
1457                         }
1458                         // if we get here, we have all the files...
1459                         return;
1460                 }
1461                 else if(!strcmp(a, "--forthismap"))
1462                 {
1463                         forthismap = true;
1464                 }
1465                 else if(!strcmp(a, "--as"))
1466                 {
1467                         if(i < end - 1)
1468                         {
1469                                 ++i;
1470                                 name = Cmd_Argv(i);
1471                         }
1472                 }
1473                 else if(!strcmp(a, "--clear_autodownload"))
1474                 {
1475                         // mark all running downloads as "not for this map", so if they
1476                         // fail, it does not matter
1477                         Curl_Clear_forthismap();
1478                         return;
1479                 }
1480                 else if(!strcmp(a, "--finish_autodownload"))
1481                 {
1482                         if(numdownloads_added)
1483                         {
1484                                 char donecommand[256];
1485                                 if(cls.netcon)
1486                                 {
1487                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1488                                         {
1489                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1490                                                 Curl_CommandWhenDone(donecommand);
1491                                                 noclear = TRUE;
1492                                                 CL_Disconnect();
1493                                                 noclear = FALSE;
1494                                                 Curl_CheckCommandWhenDone();
1495                                         }
1496                                         else
1497                                                 Curl_Register_predownload();
1498                                 }
1499                         }
1500                         return;
1501                 }
1502                 else if(!strncmp(a, "--maxspeed=", 11))
1503                 {
1504                         maxspeed = atof(a + 11);
1505                 }
1506                 else if(*a == '-')
1507                 {
1508                         Con_Printf("curl: invalid option %s\n", a);
1509                         // but we ignore the option
1510                 }
1511         }
1512
1513 needthefile:
1514         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1515 }
1516
1517 /*
1518 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1519 {
1520         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1521         Z_Free(buffer);
1522 }
1523
1524 void Curl_CurlCat_f(void)
1525 {
1526         unsigned char *buf;
1527         const char *url = Cmd_Argv(1);
1528         buf = Z_Malloc(16384);
1529         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1530 }
1531 */
1532
1533 /*
1534 ====================
1535 Curl_Init_Commands
1536
1537 loads the commands and cvars this library uses
1538 ====================
1539 */
1540 void Curl_Init_Commands(void)
1541 {
1542         Cvar_RegisterVariable (&cl_curl_enabled);
1543         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1544         Cvar_RegisterVariable (&cl_curl_maxspeed);
1545         Cvar_RegisterVariable (&sv_curl_defaulturl);
1546         Cvar_RegisterVariable (&sv_curl_serverpackages);
1547         Cvar_RegisterVariable (&sv_curl_maxspeed);
1548         Cvar_RegisterVariable (&cl_curl_useragent);
1549         Cvar_RegisterVariable (&cl_curl_useragent_append);
1550         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1551         //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1552 }
1553
1554 /*
1555 ====================
1556 Curl_GetDownloadInfo
1557
1558 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1559 The number of elements in the array is returned in int *nDownloads.
1560 const char **additional_info may be set to a string of additional user
1561 information, or to NULL if no such display shall occur. The returned
1562 array must be freed later using Z_Free.
1563 ====================
1564 */
1565 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1566 {
1567         int i;
1568         downloadinfo *di;
1569         Curl_downloadinfo_t *downinfo;
1570
1571         if(!curl_dll)
1572         {
1573                 *nDownloads = 0;
1574                 if(additional_info)
1575                         *additional_info = NULL;
1576                 return NULL;
1577         }
1578
1579         if (curl_mutex) Thread_LockMutex(curl_mutex);
1580
1581         i = 0;
1582         for(di = downloads; di; di = di->next)
1583                 ++i;
1584
1585         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1586         i = 0;
1587         for(di = downloads; di; di = di->next)
1588         {
1589                 // do not show infobars for background downloads
1590                 if(developer.integer <= 0)
1591                         if(di->buffer)
1592                                 continue;
1593                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1594                 if(di->curle)
1595                 {
1596                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1597                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1598                         downinfo[i].queued = false;
1599                 }
1600                 else
1601                 {
1602                         downinfo[i].queued = true;
1603                 }
1604                 ++i;
1605         }
1606
1607         if(additional_info)
1608         {
1609                 // TODO: can I clear command_when_done as soon as the first download fails?
1610                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1611                 {
1612                         if(!strncmp(command_when_done, "connect ", 8))
1613                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1614                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1615                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1616                         else
1617                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1618                         *additional_info = addinfo;
1619                 }
1620                 else
1621                         *additional_info = NULL;
1622         }
1623
1624         *nDownloads = i;
1625         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1626         return downinfo;
1627 }
1628
1629
1630 /*
1631 ====================
1632 Curl_FindPackURL
1633
1634 finds the URL where to find a given package.
1635
1636 For this, it reads a file "curl_urls.txt" of the following format:
1637
1638         data*.pk3       -
1639         revdm*.pk3      http://revdm/downloads/are/here/
1640         *                       http://any/other/stuff/is/here/
1641
1642 The URLs should end in /. If not, downloads will still work, but the cached files
1643 can't be just put into the data directory with the same download configuration
1644 (you might want to do this if you want to tag downloaded files from your
1645 server, but you should not). "-" means "don't download".
1646
1647 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1648 location instead.
1649
1650 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1651 this file for obvious reasons.
1652 ====================
1653 */
1654 static const char *Curl_FindPackURL(const char *filename)
1655 {
1656         static char foundurl[1024]; // invoked only by server
1657         fs_offset_t filesize;
1658         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1659         if(buf && filesize)
1660         {
1661                 // read lines of format "pattern url"
1662                 char *p = buf;
1663                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1664                 qboolean eof = false;
1665
1666                 pattern = p;
1667                 while(!eof)
1668                 {
1669                         switch(*p)
1670                         {
1671                                 case 0:
1672                                         eof = true;
1673                                         // fallthrough
1674                                 case '\n':
1675                                 case '\r':
1676                                         if(pattern && url && patternend)
1677                                         {
1678                                                 if(!urlend)
1679                                                         urlend = p;
1680                                                 *patternend = 0;
1681                                                 *urlend = 0;
1682                                                 if(matchpattern(filename, pattern, true))
1683                                                 {
1684                                                         strlcpy(foundurl, url, sizeof(foundurl));
1685                                                         Z_Free(buf);
1686                                                         return foundurl;
1687                                                 }
1688                                         }
1689                                         pattern = NULL;
1690                                         patternend = NULL;
1691                                         url = NULL;
1692                                         urlend = NULL;
1693                                         break;
1694                                 case ' ':
1695                                 case '\t':
1696                                         if(pattern && !patternend)
1697                                                 patternend = p;
1698                                         else if(url && !urlend)
1699                                                 urlend = p;
1700                                         break;
1701                                 default:
1702                                         if(!pattern)
1703                                                 pattern = p;
1704                                         else if(pattern && patternend && !url)
1705                                                 url = p;
1706                                         break;
1707                         }
1708                         ++p;
1709                 }
1710         }
1711         if(buf)
1712                 Z_Free(buf);
1713         return sv_curl_defaulturl.string;
1714 }
1715
1716 typedef struct requirement_s
1717 {
1718         struct requirement_s *next;
1719         char filename[MAX_OSPATH];
1720 }
1721 requirement;
1722 static requirement *requirements = NULL;
1723
1724
1725 /*
1726 ====================
1727 Curl_RequireFile
1728
1729 Adds the given file to the list of requirements.
1730 ====================
1731 */
1732 void Curl_RequireFile(const char *filename)
1733 {
1734         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1735         req->next = requirements;
1736         strlcpy(req->filename, filename, sizeof(req->filename));
1737         requirements = req;
1738 }
1739
1740 /*
1741 ====================
1742 Curl_ClearRequirements
1743
1744 Clears the list of required files for playing on the current map.
1745 This should be called at every map change.
1746 ====================
1747 */
1748 void Curl_ClearRequirements(void)
1749 {
1750         while(requirements)
1751         {
1752                 requirement *req = requirements;
1753                 requirements = requirements->next;
1754                 Z_Free(req);
1755         }
1756 }
1757
1758 /*
1759 ====================
1760 Curl_SendRequirements
1761
1762 Makes the current host_clients download all files he needs.
1763 This is done by sending him the following console commands:
1764
1765         curl --clear_autodownload
1766         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1767         curl --finish_autodownload
1768 ====================
1769 */
1770 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1771 {
1772         const char *p;
1773         const char *thispack = FS_WhichPack(filename);
1774         const char *packurl;
1775
1776         if(!thispack)
1777                 return false;
1778
1779         p = strrchr(thispack, '/');
1780         if(p)
1781                 thispack = p + 1;
1782
1783         packurl = Curl_FindPackURL(thispack);
1784
1785         if(packurl && *packurl && strcmp(packurl, "-"))
1786         {
1787                 if(!foundone)
1788                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1789
1790                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1791                 strlcat(sendbuffer, thispack, sendbuffer_len);
1792                 if(sv_curl_maxspeed.value > 0)
1793                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1794                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1795                 strlcat(sendbuffer, filename, sendbuffer_len);
1796                 strlcat(sendbuffer, " ", sendbuffer_len);
1797                 strlcat(sendbuffer, packurl, sendbuffer_len);
1798                 strlcat(sendbuffer, thispack, sendbuffer_len);
1799                 strlcat(sendbuffer, "\n", sendbuffer_len);
1800
1801                 return true;
1802         }
1803
1804         return false;
1805 }
1806 void Curl_SendRequirements(void)
1807 {
1808         // for each requirement, find the pack name
1809         char sendbuffer[4096] = "";
1810         requirement *req;
1811         qboolean foundone = false;
1812         const char *p;
1813
1814         for(req = requirements; req; req = req->next)
1815                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1816
1817         p = sv_curl_serverpackages.string;
1818         while(COM_ParseToken_Simple(&p, false, false, true))
1819                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1820
1821         if(foundone)
1822                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1823
1824         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1825                 Host_ClientCommands("%s", sendbuffer);
1826         else
1827                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1828 }