libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5
   6 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   7 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
   8 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
   9 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  10 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  11 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  12 static cvar_t cl_curl_useragent = {0, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  13 static cvar_t cl_curl_useragent_append = {0, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  14
  15 /*
  16 =================================================================
  17
  18   Minimal set of definitions from libcurl
  19
  20   WARNING: for a matter of simplicity, several pointer types are
  21   casted to "void*", and most enumerated values are not included
  22
  23 =================================================================
  24 */
  25
  26 typedef struct CURL_s CURL;
  27 typedef struct CURLM_s CURLM;
  28 typedef struct curl_slist curl_slist;
  29 typedef enum
  30 {
  31         CURLE_OK = 0
  32 }
  33 CURLcode;
  34 typedef enum
  35 {
  36         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  37         CURLM_OK = 0
  38 }
  39 CURLMcode;
  40 #define CURL_GLOBAL_NOTHING 0
  41 #define CURL_GLOBAL_SSL 1
  42 #define CURL_GLOBAL_WIN32 2
  43 #define CURLOPTTYPE_LONG          0
  44 #define CURLOPTTYPE_OBJECTPOINT   10000
  45 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  46 #define CURLOPTTYPE_OFF_T         30000
  47 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  48 typedef enum
  49 {
  50         CINIT(WRITEDATA, OBJECTPOINT, 1),
  51         CINIT(URL,  OBJECTPOINT, 2),
  52         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  53         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  54         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  55         CINIT(REFERER, OBJECTPOINT, 16),
  56         CINIT(USERAGENT, OBJECTPOINT, 18),
  57         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  58         CINIT(LOW_SPEED_TIME, LONG, 20),
  59         CINIT(RESUME_FROM, LONG, 21),
  60         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  61         CINIT(POST, LONG, 47),         /* HTTP POST method */
  62         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  63         CINIT(POSTFIELDSIZE, LONG, 60),
  64         CINIT(PRIVATE, OBJECTPOINT, 103),
  65         CINIT(PROTOCOLS, LONG, 181),
  66         CINIT(REDIR_PROTOCOLS, LONG, 182)
  67 }
  68 CURLoption;
  69 #define CURLPROTO_HTTP   (1<<0)
  70 #define CURLPROTO_HTTPS  (1<<1)
  71 #define CURLPROTO_FTP    (1<<2)
  72 typedef enum
  73 {
  74         CURLINFO_TEXT = 0,
  75         CURLINFO_HEADER_IN,    /* 1 */
  76         CURLINFO_HEADER_OUT,   /* 2 */
  77         CURLINFO_DATA_IN,      /* 3 */
  78         CURLINFO_DATA_OUT,     /* 4 */
  79         CURLINFO_SSL_DATA_IN,  /* 5 */
  80         CURLINFO_SSL_DATA_OUT, /* 6 */
  81         CURLINFO_END
  82 }
  83 curl_infotype;
  84 #define CURLINFO_STRING   0x100000
  85 #define CURLINFO_LONG     0x200000
  86 #define CURLINFO_DOUBLE   0x300000
  87 #define CURLINFO_SLIST    0x400000
  88 #define CURLINFO_MASK     0x0fffff
  89 #define CURLINFO_TYPEMASK 0xf00000
  90 typedef enum
  91 {
  92         CURLINFO_NONE, /* first, never use this */
  93         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  94         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  95         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  96         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  97         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  98         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  99         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 100         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 101         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 102         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 103         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 104         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 105         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 106         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 107         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 108         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 109         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 110         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 111         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 112         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 113         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 114         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 115         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 116         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 117         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 118         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 119         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 120 }
 121 CURLINFO;
 122
 123 typedef enum
 124 {
 125         CURLMSG_NONE, /* first, not used */
 126         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 127                                          the CURLcode of the transfer */
 128         CURLMSG_LAST
 129 }
 130 CURLMSG;
 131 typedef struct
 132 {
 133         CURLMSG msg;       /* what this message means */
 134         CURL *easy_handle; /* the handle it concerns */
 135         union
 136         {
 137                 void *whatever;    /* message-specific data */
 138                 CURLcode result;   /* return code for transfer */
 139         }
 140         data;
 141 }
 142 CURLMsg;
 143
 144 static void (*qcurl_global_init) (long flags);
 145 static void (*qcurl_global_cleanup) (void);
 146
 147 static CURL * (*qcurl_easy_init) (void);
 148 static void (*qcurl_easy_cleanup) (CURL *handle);
 149 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 150 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 151 static const char * (*qcurl_easy_strerror) (CURLcode);
 152
 153 static CURLM * (*qcurl_multi_init) (void);
 154 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 155 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 156 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 157 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 158 static void (*qcurl_multi_cleanup) (CURLM *);
 159 static const char * (*qcurl_multi_strerror) (CURLcode);
 160 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 161 static void (*qcurl_slist_free_all) (curl_slist *list);
 162
 163 static dllfunction_t curlfuncs[] =
 164 {
 165         {"curl_global_init",            (void **) &qcurl_global_init},
 166         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 167         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 168         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 169         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 170         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 171         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 172         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 173         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 174         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 175         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 176         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 177         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 178         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 179         {"curl_slist_append",           (void **) &qcurl_slist_append},
 180         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 181         {NULL, NULL}
 182 };
 183
 184 // Handle for CURL DLL
 185 static dllhandle_t curl_dll = NULL;
 186 // will be checked at many places to find out if qcurl calls are allowed
 187
 188 void *curl_mutex = NULL;
 189
 190 typedef struct downloadinfo_s
 191 {
 192         char filename[MAX_OSPATH];
 193         char url[1024];
 194         char referer[256];
 195         qfile_t *stream;
 196         fs_offset_t startpos;
 197         CURL *curle;
 198         qboolean started;
 199         qboolean ispak;
 200         unsigned long bytes_received; // for buffer
 201         double bytes_received_curl; // for throttling
 202         double bytes_sent_curl; // for throttling
 203         struct downloadinfo_s *next, *prev;
 204         qboolean forthismap;
 205         double maxspeed;
 206         curl_slist *slist; // http headers
 207
 208         unsigned char *buffer;
 209         size_t buffersize;
 210         curl_callback_t callback;
 211         void *callback_data;
 212
 213         const unsigned char *postbuf;
 214         size_t postbufsize;
 215         const char *post_content_type;
 216         const char *extraheaders;
 217 }
 218 downloadinfo;
 219 static downloadinfo *downloads = NULL;
 220 static int numdownloads = 0;
 221
 222 static qboolean noclear = FALSE;
 223
 224 static int numdownloads_fail = 0;
 225 static int numdownloads_success = 0;
 226 static int numdownloads_added = 0;
 227 static char command_when_done[256] = "";
 228 static char command_when_error[256] = "";
 229
 230 /*
 231 ====================
 232 Curl_CommandWhenDone
 233
 234 Sets the command which is to be executed when the last download completes AND
 235 all downloads since last server connect ended with a successful status.
 236 Setting the command to NULL clears it.
 237 ====================
 238 */
 239 static void Curl_CommandWhenDone(const char *cmd)
 240 {
 241         if(!curl_dll)
 242                 return;
 243         if(cmd)
 244                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 245         else
 246                 *command_when_done = 0;
 247 }
 248
 249 /*
 250 FIXME
 251 Do not use yet. Not complete.
 252 Problem: what counts as an error?
 253 */
 254
 255 static void Curl_CommandWhenError(const char *cmd)
 256 {
 257         if(!curl_dll)
 258                 return;
 259         if(cmd)
 260                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 261         else
 262                 *command_when_error = 0;
 263 }
 264
 265 /*
 266 ====================
 267 Curl_Clear_forthismap
 268
 269 Clears the "will disconnect on failure" flags.
 270 ====================
 271 */
 272 void Curl_Clear_forthismap(void)
 273 {
 274         downloadinfo *di;
 275         if(noclear)
 276                 return;
 277         if (curl_mutex) Thread_LockMutex(curl_mutex);
 278         for(di = downloads; di; di = di->next)
 279                 di->forthismap = false;
 280         Curl_CommandWhenError(NULL);
 281         Curl_CommandWhenDone(NULL);
 282         numdownloads_fail = 0;
 283         numdownloads_success = 0;
 284         numdownloads_added = 0;
 285         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 286 }
 287
 288 /*
 289 ====================
 290 Curl_Have_forthismap
 291
 292 Returns true if a download needed for the current game is running.
 293 ====================
 294 */
 295 qboolean Curl_Have_forthismap(void)
 296 {
 297         return numdownloads_added != 0;
 298 }
 299
 300 void Curl_Register_predownload(void)
 301 {
 302         if (curl_mutex) Thread_LockMutex(curl_mutex);
 303         Curl_CommandWhenDone("cl_begindownloads");
 304         Curl_CommandWhenError("cl_begindownloads");
 305         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 306 }
 307
 308 /*
 309 ====================
 310 Curl_CheckCommandWhenDone
 311
 312 Checks if a "done command" is to be executed.
 313 All downloads finished, at least one success since connect, no single failure
 314 -> execute the command.
 315 */
 316 static void Curl_CheckCommandWhenDone(void)
 317 {
 318         if(!curl_dll)
 319                 return;
 320         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 321         {
 322                 if(numdownloads_fail == 0)
 323                 {
 324                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 325                         Cbuf_AddText("\n");
 326                         Cbuf_AddText(command_when_done);
 327                         Cbuf_AddText("\n");
 328                 }
 329                 else
 330                 {
 331                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 332                         Cbuf_AddText("\n");
 333                         Cbuf_AddText(command_when_error);
 334                         Cbuf_AddText("\n");
 335                 }
 336                 Curl_Clear_forthismap();
 337         }
 338 }
 339
 340 /*
 341 ====================
 342 CURL_CloseLibrary
 343
 344 Load the cURL DLL
 345 ====================
 346 */
 347 static qboolean CURL_OpenLibrary (void)
 348 {
 349         const char* dllnames [] =
 350         {
 351 #if defined(WIN32)
 352                 "libcurl-4.dll",
 353                 "libcurl-3.dll",
 354 #elif defined(MACOSX)
 355                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 356                 "libcurl.3.dylib", // Mac OS X Tiger
 357                 "libcurl.2.dylib", // Mac OS X Panther
 358 #else
 359                 "libcurl.so.4",
 360                 "libcurl.so.3",
 361                 "libcurl.so", // FreeBSD
 362 #endif
 363                 NULL
 364         };
 365
 366         // Already loaded?
 367         if (curl_dll)
 368                 return true;
 369
 370         // Load the DLL
 371         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
 372 }
 373
 374
 375 /*
 376 ====================
 377 CURL_CloseLibrary
 378
 379 Unload the cURL DLL
 380 ====================
 381 */
 382 static void CURL_CloseLibrary (void)
 383 {
 384         Sys_UnloadLibrary (&curl_dll);
 385 }
 386
 387
 388 static CURLM *curlm = NULL;
 389 static double bytes_received = 0; // used for bandwidth throttling
 390 static double bytes_sent = 0; // used for bandwidth throttling
 391 static double curltime = 0;
 392
 393 /*
 394 ====================
 395 CURL_fwrite
 396
 397 fwrite-compatible function that writes the data to a file. libcurl can call
 398 this.
 399 ====================
 400 */
 401 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 402 {
 403         fs_offset_t ret = -1;
 404         size_t bytes = size * nmemb;
 405         downloadinfo *di = (downloadinfo *) vdi;
 406
 407         if(di->buffer)
 408         {
 409                 if(di->bytes_received + bytes <= di->buffersize)
 410                 {
 411                         memcpy(di->buffer + di->bytes_received, data, bytes);
 412                         ret = bytes;
 413                 }
 414                 // otherwise: buffer overrun, ret stays -1
 415         }
 416
 417         if(di->stream)
 418         {
 419                 ret = FS_Write(di->stream, data, bytes);
 420         }
 421
 422         di->bytes_received += bytes;
 423
 424         return ret; // why not ret / nmemb?
 425 }
 426
 427 typedef enum
 428 {
 429         CURL_DOWNLOAD_SUCCESS = 0,
 430         CURL_DOWNLOAD_FAILED,
 431         CURL_DOWNLOAD_ABORTED,
 432         CURL_DOWNLOAD_SERVERERROR
 433 }
 434 CurlStatus;
 435
 436 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 437 {
 438         downloadinfo *di = (downloadinfo *) cbdata;
 439         switch(status)
 440         {
 441                 case CURLCBSTATUS_OK:
 442                         Con_DPrintf("Download of %s: OK\n", di->filename);
 443                         break;
 444                 case CURLCBSTATUS_FAILED:
 445                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 446                         break;
 447                 case CURLCBSTATUS_ABORTED:
 448                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 449                         break;
 450                 case CURLCBSTATUS_SERVERERROR:
 451                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 452                         break;
 453                 case CURLCBSTATUS_UNKNOWN:
 454                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 455                         break;
 456                 default:
 457                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 458                         break;
 459         }
 460 }
 461
 462 /*
 463 ====================
 464 Curl_EndDownload
 465
 466 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 467 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 468 code from libcurl, or 0, if another error has occurred.
 469 ====================
 470 */
 471 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 472 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 473 {
 474         qboolean ok = false;
 475         if(!curl_dll)
 476                 return;
 477         switch(status)
 478         {
 479                 case CURL_DOWNLOAD_SUCCESS:
 480                         ok = true;
 481                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 482                         break;
 483                 case CURL_DOWNLOAD_FAILED:
 484                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 485                         break;
 486                 case CURL_DOWNLOAD_ABORTED:
 487                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 488                         break;
 489                 case CURL_DOWNLOAD_SERVERERROR:
 490                         // reopen to enforce it to have zero bytes again
 491                         if(di->stream)
 492                         {
 493                                 FS_Close(di->stream);
 494                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 495                         }
 496
 497                         if(di->callback)
 498                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 499                         break;
 500                 default:
 501                         if(di->callback)
 502                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 503                         break;
 504         }
 505
 506         if(di->curle)
 507         {
 508                 qcurl_multi_remove_handle(curlm, di->curle);
 509                 qcurl_easy_cleanup(di->curle);
 510                 if(di->slist)
 511                         qcurl_slist_free_all(di->slist);
 512         }
 513
 514         if(!di->callback && ok && !di->bytes_received)
 515         {
 516                 Con_Printf("ERROR: empty file\n");
 517                 ok = false;
 518         }
 519
 520         if(di->stream)
 521                 FS_Close(di->stream);
 522
 523         if(ok && di->ispak)
 524         {
 525                 ok = FS_AddPack(di->filename, NULL, true);
 526                 if(!ok)
 527                 {
 528                         // pack loading failed?
 529                         // this is critical
 530                         // better clear the file again...
 531                         di->stream = FS_OpenRealFile(di->filename, "wb", false);
 532                         FS_Close(di->stream);
 533
 534                         if(di->startpos && !di->callback)
 535                         {
 536                                 // this was a resume?
 537                                 // then try to redownload it without reporting the error
 538                                 Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->ispak, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL);
 539                                 di->forthismap = false; // don't count the error
 540                         }
 541                 }
 542         }
 543
 544         if(di->prev)
 545                 di->prev->next = di->next;
 546         else
 547                 downloads = di->next;
 548         if(di->next)
 549                 di->next->prev = di->prev;
 550
 551         --numdownloads;
 552         if(di->forthismap)
 553         {
 554                 if(ok)
 555                         ++numdownloads_success;
 556                 else
 557                         ++numdownloads_fail;
 558         }
 559         Z_Free(di);
 560 }
 561
 562 /*
 563 ====================
 564 CleanURL
 565
 566 Returns a "cleaned up" URL for display (to strip login data)
 567 ====================
 568 */
 569 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 570 {
 571         const char *p, *q, *r;
 572
 573         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 574         p = strstr(url, "://");
 575         if(p)
 576         {
 577                 q = strchr(p + 3, '@');
 578                 if(q)
 579                 {
 580                         r = strchr(p + 3, '/');
 581                         if(!r || q < r)
 582                         {
 583                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 584                                 return urlbuf;
 585                         }
 586                 }
 587         }
 588
 589         return url;
 590 }
 591
 592 /*
 593 ====================
 594 CheckPendingDownloads
 595
 596 checks if there are free download slots to start new downloads in.
 597 To not start too many downloads at once, only one download is added at a time,
 598 up to a maximum number of cl_curl_maxdownloads are running.
 599 ====================
 600 */
 601 static void CheckPendingDownloads(void)
 602 {
 603         const char *h;
 604         char urlbuf[1024];
 605         char vabuf[1024];
 606         if(!curl_dll)
 607                 return;
 608         if(numdownloads < cl_curl_maxdownloads.integer)
 609         {
 610                 downloadinfo *di;
 611                 for(di = downloads; di; di = di->next)
 612                 {
 613                         if(!di->started)
 614                         {
 615                                 if(!di->buffer)
 616                                 {
 617                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 618
 619                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 620                                         if(!di->stream)
 621                                         {
 622                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 623                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 624                                                 return;
 625                                         }
 626                                         FS_Seek(di->stream, 0, SEEK_END);
 627                                         di->startpos = FS_Tell(di->stream);
 628
 629                                         if(di->startpos > 0)
 630                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 631                                         Con_Print("...\n");
 632                                 }
 633                                 else
 634                                 {
 635                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 636                                         di->startpos = 0;
 637                                 }
 638
 639                                 di->curle = qcurl_easy_init();
 640                                 di->slist = NULL;
 641                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 642                                 if(cl_curl_useragent.integer)
 643                                 {
 644                                         const char *ua
 645 #ifdef HTTP_USER_AGENT
 646                                                 = HTTP_USER_AGENT;
 647 #else
 648                                                 = engineversion;
 649 #endif
 650                                         if(!ua)
 651                                                 ua = "";
 652                                         if(*cl_curl_useragent_append.string)
 653                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 654                                                         ua,
 655                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 656                                                                 ? " "
 657                                                                 : "",
 658                                                         cl_curl_useragent_append.string);
 659                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 660                                 }
 661                                 else
 662                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 663                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 664                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 665                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 666                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 667                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 668                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 669                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 670                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 671                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 672                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 673                                 {
 674                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 675                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 676                                 }
 677                                 if(di->post_content_type)
 678                                 {
 679                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 680                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 681                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 682                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 683                                 }
 684
 685                                 // parse extra headers into slist
 686                                 // \n separated list!
 687                                 h = di->extraheaders;
 688                                 while(h)
 689                                 {
 690                                         const char *hh = strchr(h, '\n');
 691                                         if(hh)
 692                                         {
 693                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 694                                                 memcpy(buf, h, hh - h);
 695                                                 buf[hh - h] = 0;
 696                                                 di->slist = qcurl_slist_append(di->slist, buf);
 697                                                 h = hh + 1;
 698                                         }
 699                                         else
 700                                         {
 701                                                 di->slist = qcurl_slist_append(di->slist, h);
 702                                                 h = NULL;
 703                                         }
 704                                 }
 705
 706                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 707
 708                                 qcurl_multi_add_handle(curlm, di->curle);
 709                                 di->started = true;
 710                                 ++numdownloads;
 711                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 712                                         break;
 713                         }
 714                 }
 715         }
 716 }
 717
 718 /*
 719 ====================
 720 Curl_Init
 721
 722 this function MUST be called before using anything else in this file.
 723 On Win32, this must be called AFTER WSAStartup has been done!
 724 ====================
 725 */
 726 void Curl_Init(void)
 727 {
 728         CURL_OpenLibrary();
 729         if(!curl_dll)
 730                 return;
 731         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 732         qcurl_global_init(CURL_GLOBAL_NOTHING);
 733         curlm = qcurl_multi_init();
 734 }
 735
 736 /*
 737 ====================
 738 Curl_Shutdown
 739
 740 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 741 ====================
 742 */
 743 void Curl_ClearRequirements(void);
 744 void Curl_Shutdown(void)
 745 {
 746         if(!curl_dll)
 747                 return;
 748         Curl_ClearRequirements();
 749         Curl_CancelAll();
 750         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 751         CURL_CloseLibrary();
 752         curl_dll = NULL;
 753 }
 754
 755 /*
 756 ====================
 757 Curl_Find
 758
 759 Finds the internal information block for a download given by file name.
 760 ====================
 761 */
 762 static downloadinfo *Curl_Find(const char *filename)
 763 {
 764         downloadinfo *di;
 765         if(!curl_dll)
 766                 return NULL;
 767         for(di = downloads; di; di = di->next)
 768                 if(!strcasecmp(di->filename, filename))
 769                         return di;
 770         return NULL;
 771 }
 772
 773 /*
 774 ====================
 775 Curl_Begin
 776
 777 Starts a download of a given URL to the file name portion of this URL (or name
 778 if given) in the "dlcache/" folder.
 779 ====================
 780 */
 781 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 782 {
 783         if(!curl_dll)
 784         {
 785                 return false;
 786         }
 787         else
 788         {
 789                 char fn[MAX_OSPATH];
 790                 char urlbuf[1024];
 791                 const char *p, *q;
 792                 size_t length;
 793                 downloadinfo *di;
 794
 795                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 796                 p = strchr(URL, ':');
 797                 if(p)
 798                 {
 799                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 800                         {
 801                                 char addressstring[128];
 802                                 *addressstring = 0;
 803                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 804                                 q = strchr(addressstring, ':');
 805                                 if(!q)
 806                                         q = addressstring + strlen(addressstring);
 807                                 if(*addressstring)
 808                                 {
 809                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 810                                         URL = urlbuf;
 811                                 }
 812                         }
 813                 }
 814
 815                 // Note: This extraction of the file name portion is NOT entirely correct.
 816                 //
 817                 // It does the following:
 818                 //
 819                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 820                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 821                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 822                 //
 823                 // However, I'd like to keep this "buggy" behavior so that PHP script
 824                 // authors can write download scripts without having to enable
 825                 // AcceptPathInfo on Apache. They just have to ensure that their script
 826                 // can be called with such a "fake" path name like
 827                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 828                 //
 829                 // By the way, such PHP scripts should either send the file or a
 830                 // "Location:" redirect; PHP code example:
 831                 //
 832                 //   header("Location: http://www.example.com/");
 833                 //
 834                 // By the way, this will set User-Agent to something like
 835                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 836                 // dp://serverhost:serverport/ so you can filter on this; an example
 837                 // httpd log file line might be:
 838                 //
 839                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 840
 841                 if(!name)
 842                         name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 843
 844                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 845
 846                 if(!buf)
 847                 {
 848                         p = strrchr(name, '/');
 849                         p = p ? (p+1) : name;
 850                         q = strchr(p, '?');
 851                         length = q ? (size_t)(q - p) : strlen(p);
 852                         dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 853
 854                         name = fn; // make it point back
 855
 856                         // already downloading the file?
 857                         {
 858                                 downloadinfo *di = Curl_Find(fn);
 859                                 if(di)
 860                                 {
 861                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 862
 863                                         // however, if it was not for this map yet...
 864                                         if(forthismap && !di->forthismap)
 865                                         {
 866                                                 di->forthismap = true;
 867                                                 // this "fakes" a download attempt so the client will wait for
 868                                                 // the download to finish and then reconnect
 869                                                 ++numdownloads_added;
 870                                         }
 871
 872                                         return false;
 873                                 }
 874                         }
 875
 876                         if(ispak && FS_FileExists(fn))
 877                         {
 878                                 qboolean already_loaded;
 879                                 if(FS_AddPack(fn, &already_loaded, true))
 880                                 {
 881                                         Con_DPrintf("%s already exists, not downloading!\n", fn);
 882                                         if(already_loaded)
 883                                                 Con_DPrintf("(pak was already loaded)\n");
 884                                         else
 885                                         {
 886                                                 if(forthismap)
 887                                                 {
 888                                                         ++numdownloads_added;
 889                                                         ++numdownloads_success;
 890                                                 }
 891                                         }
 892
 893                                         return false;
 894                                 }
 895                                 else
 896                                 {
 897                                         qfile_t *f = FS_OpenRealFile(fn, "rb", false);
 898                                         if(f)
 899                                         {
 900                                                 char buf[4] = {0};
 901                                                 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 902
 903                                                 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 904                                                 {
 905                                                         Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 906                                                         FS_Close(f);
 907                                                         f = FS_OpenRealFile(fn, "wb", false);
 908                                                         if(f)
 909                                                                 FS_Close(f);
 910                                                 }
 911                                                 else
 912                                                 {
 913                                                         // OK
 914                                                         FS_Close(f);
 915                                                 }
 916                                         }
 917                                 }
 918                         }
 919                 }
 920
 921                 // if we get here, we actually want to download... so first verify the
 922                 // URL scheme (so one can't read local files using file://)
 923                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
 924                 {
 925                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
 926                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 927                         return false;
 928                 }
 929
 930                 if(forthismap)
 931                         ++numdownloads_added;
 932                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 933                 strlcpy(di->filename, name, sizeof(di->filename));
 934                 strlcpy(di->url, URL, sizeof(di->url));
 935                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 936                 di->forthismap = forthismap;
 937                 di->stream = NULL;
 938                 di->startpos = 0;
 939                 di->curle = NULL;
 940                 di->started = false;
 941                 di->ispak = (ispak && !buf);
 942                 di->maxspeed = maxspeed;
 943                 di->bytes_received = 0;
 944                 di->bytes_received_curl = 0;
 945                 di->bytes_sent_curl = 0;
 946                 di->extraheaders = extraheaders;
 947                 di->next = downloads;
 948                 di->prev = NULL;
 949                 if(di->next)
 950                         di->next->prev = di;
 951
 952                 di->buffer = buf;
 953                 di->buffersize = bufsize;
 954                 if(callback == NULL)
 955                 {
 956                         di->callback = curl_default_callback;
 957                         di->callback_data = di;
 958                 }
 959                 else
 960                 {
 961                         di->callback = callback;
 962                         di->callback_data = cbdata;
 963                 }
 964
 965                 if(post_content_type)
 966                 {
 967                         di->post_content_type = post_content_type;
 968                         di->postbuf = postbuf;
 969                         di->postbufsize = postbufsize;
 970                 }
 971                 else
 972                 {
 973                         di->post_content_type = NULL;
 974                         di->postbuf = NULL;
 975                         di->postbufsize = 0;
 976                 }
 977
 978                 downloads = di;
 979                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 980                 return true;
 981         }
 982 }
 983
 984 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap)
 985 {
 986         return Curl_Begin(URL, NULL, maxspeed, name, ispak, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
 987 }
 988 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 989 {
 990         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
 991 }
 992 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 993 {
 994         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
 995 }
 996
 997 /*
 998 ====================
 999 Curl_Run
1000
1001 call this regularily as this will always download as much as possible without
1002 blocking.
1003 ====================
1004 */
1005 void Curl_Run(void)
1006 {
1007         double maxspeed;
1008         downloadinfo *di;
1009
1010         noclear = FALSE;
1011
1012         if(!cl_curl_enabled.integer)
1013                 return;
1014
1015         if(!curl_dll)
1016                 return;
1017
1018         if (curl_mutex) Thread_LockMutex(curl_mutex);
1019
1020         Curl_CheckCommandWhenDone();
1021
1022         if(!downloads)
1023         {
1024                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1025                 return;
1026         }
1027
1028         if(realtime < curltime) // throttle
1029         {
1030                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1031                 return;
1032         }
1033
1034         {
1035                 int remaining;
1036                 CURLMcode mc;
1037
1038                 do
1039                 {
1040                         mc = qcurl_multi_perform(curlm, &remaining);
1041                 }
1042                 while(mc == CURLM_CALL_MULTI_PERFORM);
1043
1044                 for(di = downloads; di; di = di->next)
1045                 {
1046                         double b = 0;
1047                         if(di->curle)
1048                         {
1049                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1050                                 bytes_sent += (b - di->bytes_sent_curl);
1051                                 di->bytes_sent_curl = b;
1052                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1053                                 bytes_sent += (b - di->bytes_received_curl);
1054                                 di->bytes_received_curl = b;
1055                         }
1056                 }
1057
1058                 for(;;)
1059                 {
1060                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1061                         if(!msg)
1062                                 break;
1063                         if(msg->msg == CURLMSG_DONE)
1064                         {
1065                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1066                                 CURLcode result;
1067                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1068                                 result = msg->data.result;
1069                                 if(result)
1070                                 {
1071                                         failed = CURL_DOWNLOAD_FAILED;
1072                                 }
1073                                 else
1074                                 {
1075                                         long code;
1076                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1077                                         switch(code / 100)
1078                                         {
1079                                                 case 4: // e.g. 404?
1080                                                 case 5: // e.g. 500?
1081                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1082                                                         result = (CURLcode) code;
1083                                                         break;
1084                                         }
1085                                 }
1086
1087                                 Curl_EndDownload(di, failed, result);
1088                         }
1089                 }
1090         }
1091
1092         CheckPendingDownloads();
1093
1094         // when will we curl the next time?
1095         // we will wait a bit to ensure our download rate is kept.
1096         // we now know that realtime >= curltime... so set up a new curltime
1097
1098         // use the slowest allowing download to derive the maxspeed... this CAN
1099         // be done better, but maybe later
1100         maxspeed = cl_curl_maxspeed.value;
1101         for(di = downloads; di; di = di->next)
1102                 if(di->maxspeed > 0)
1103                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1104                                 maxspeed = di->maxspeed;
1105
1106         if(maxspeed > 0)
1107         {
1108                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1109                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1110                 bytes_sent = 0;
1111                 bytes_received = 0;
1112         }
1113         else
1114                 curltime = realtime;
1115
1116         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1117 }
1118
1119 /*
1120 ====================
1121 Curl_CancelAll
1122
1123 Stops ALL downloads.
1124 ====================
1125 */
1126 void Curl_CancelAll(void)
1127 {
1128         if(!curl_dll)
1129                 return;
1130
1131         if (curl_mutex) Thread_LockMutex(curl_mutex);
1132
1133         while(downloads)
1134         {
1135                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1136                 // INVARIANT: downloads will point to the next download after that!
1137         }
1138
1139         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1140 }
1141
1142 /*
1143 ====================
1144 Curl_Running
1145
1146 returns true iff there is a download running.
1147 ====================
1148 */
1149 qboolean Curl_Running(void)
1150 {
1151         if(!curl_dll)
1152                 return false;
1153
1154         return downloads != NULL;
1155 }
1156
1157 /*
1158 ====================
1159 Curl_GetDownloadAmount
1160
1161 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1162 for the given download.
1163 ====================
1164 */
1165 static double Curl_GetDownloadAmount(downloadinfo *di)
1166 {
1167         if(!curl_dll)
1168                 return -2;
1169         if(di->curle)
1170         {
1171                 double length;
1172                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1173                 if(length > 0)
1174                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1175                 else
1176                         return 0;
1177         }
1178         else
1179                 return -1;
1180 }
1181
1182 /*
1183 ====================
1184 Curl_GetDownloadSpeed
1185
1186 returns the speed of the given download in bytes per second
1187 ====================
1188 */
1189 static double Curl_GetDownloadSpeed(downloadinfo *di)
1190 {
1191         if(!curl_dll)
1192                 return -2;
1193         if(di->curle)
1194         {
1195                 double speed;
1196                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1197                 return speed;
1198         }
1199         else
1200                 return -1;
1201 }
1202
1203 /*
1204 ====================
1205 Curl_Info_f
1206
1207 prints the download list
1208 ====================
1209 */
1210 // TODO rewrite using Curl_GetDownloadInfo?
1211 static void Curl_Info_f(void)
1212 {
1213         downloadinfo *di;
1214         char urlbuf[1024];
1215         if(!curl_dll)
1216                 return;
1217         if(Curl_Running())
1218         {
1219                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1220                 Con_Print("Currently running downloads:\n");
1221                 for(di = downloads; di; di = di->next)
1222                 {
1223                         double speed, percent;
1224                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1225                         percent = 100.0 * Curl_GetDownloadAmount(di);
1226                         speed = Curl_GetDownloadSpeed(di);
1227                         if(percent >= 0)
1228                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1229                         else
1230                                 Con_Print("(queued)\n");
1231                 }
1232                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1233         }
1234         else
1235         {
1236                 Con_Print("No downloads running.\n");
1237         }
1238 }
1239
1240 /*
1241 ====================
1242 Curl_Curl_f
1243
1244 implements the "curl" console command
1245
1246 curl --info
1247 curl --cancel
1248 curl --cancel filename
1249 curl url
1250
1251 For internal use:
1252
1253 curl [--pak] [--forthismap] [--for filename filename...] url
1254         --pak: after downloading, load the package into the virtual file system
1255         --for filename...: only download of at least one of the named files is missing
1256         --forthismap: don't reconnect on failure
1257
1258 curl --clear_autodownload
1259         clears the download success/failure counters
1260
1261 curl --finish_autodownload
1262         if at least one download has been started, disconnect and drop to the menu
1263         once the last download completes successfully, reconnect to the current server
1264 ====================
1265 */
1266 static void Curl_Curl_f(void)
1267 {
1268         double maxspeed = 0;
1269         int i;
1270         int end;
1271         qboolean pak = false;
1272         qboolean forthismap = false;
1273         const char *url;
1274         const char *name = 0;
1275
1276         if(!curl_dll)
1277         {
1278                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1279                 return;
1280         }
1281
1282         if(!cl_curl_enabled.integer)
1283         {
1284                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1285                 return;
1286         }
1287
1288         if(Cmd_Argc() < 2)
1289         {
1290                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1291                 return;
1292         }
1293
1294         url = Cmd_Argv(Cmd_Argc() - 1);
1295         end = Cmd_Argc();
1296
1297         for(i = 1; i != end; ++i)
1298         {
1299                 const char *a = Cmd_Argv(i);
1300                 if(!strcmp(a, "--info"))
1301                 {
1302                         Curl_Info_f();
1303                         return;
1304                 }
1305                 else if(!strcmp(a, "--cancel"))
1306                 {
1307                         if(i == end - 1) // last argument
1308                                 Curl_CancelAll();
1309                         else
1310                         {
1311                                 downloadinfo *di = Curl_Find(url);
1312                                 if(di)
1313                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1314                                 else
1315                                         Con_Print("download not found\n");
1316                         }
1317                         return;
1318                 }
1319                 else if(!strcmp(a, "--pak"))
1320                 {
1321                         pak = true;
1322                 }
1323                 else if(!strcmp(a, "--for")) // must be last option
1324                 {
1325                         for(i = i + 1; i != end - 1; ++i)
1326                         {
1327                                 if(!FS_FileExists(Cmd_Argv(i)))
1328                                         goto needthefile; // why can't I have a "double break"?
1329                         }
1330                         // if we get here, we have all the files...
1331                         return;
1332                 }
1333                 else if(!strcmp(a, "--forthismap"))
1334                 {
1335                         forthismap = true;
1336                 }
1337                 else if(!strcmp(a, "--as"))
1338                 {
1339                         if(i < end - 1)
1340                         {
1341                                 ++i;
1342                                 name = Cmd_Argv(i);
1343                         }
1344                 }
1345                 else if(!strcmp(a, "--clear_autodownload"))
1346                 {
1347                         // mark all running downloads as "not for this map", so if they
1348                         // fail, it does not matter
1349                         Curl_Clear_forthismap();
1350                         return;
1351                 }
1352                 else if(!strcmp(a, "--finish_autodownload"))
1353                 {
1354                         if(numdownloads_added)
1355                         {
1356                                 char donecommand[256];
1357                                 if(cls.netcon)
1358                                 {
1359                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1360                                         {
1361                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1362                                                 Curl_CommandWhenDone(donecommand);
1363                                                 noclear = TRUE;
1364                                                 CL_Disconnect();
1365                                                 noclear = FALSE;
1366                                                 Curl_CheckCommandWhenDone();
1367                                         }
1368                                         else
1369                                                 Curl_Register_predownload();
1370                                 }
1371                         }
1372                         return;
1373                 }
1374                 else if(!strncmp(a, "--maxspeed=", 11))
1375                 {
1376                         maxspeed = atof(a + 11);
1377                 }
1378                 else if(*a == '-')
1379                 {
1380                         Con_Printf("curl: invalid option %s\n", a);
1381                         // but we ignore the option
1382                 }
1383         }
1384
1385 needthefile:
1386         Curl_Begin_ToFile(url, maxspeed, name, pak, forthismap);
1387 }
1388
1389 /*
1390 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1391 {
1392         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1393         Z_Free(buffer);
1394 }
1395
1396 void Curl_CurlCat_f(void)
1397 {
1398         unsigned char *buf;
1399         const char *url = Cmd_Argv(1);
1400         buf = Z_Malloc(16384);
1401         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1402 }
1403 */
1404
1405 /*
1406 ====================
1407 Curl_Init_Commands
1408
1409 loads the commands and cvars this library uses
1410 ====================
1411 */
1412 void Curl_Init_Commands(void)
1413 {
1414         Cvar_RegisterVariable (&cl_curl_enabled);
1415         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1416         Cvar_RegisterVariable (&cl_curl_maxspeed);
1417         Cvar_RegisterVariable (&sv_curl_defaulturl);
1418         Cvar_RegisterVariable (&sv_curl_serverpackages);
1419         Cvar_RegisterVariable (&sv_curl_maxspeed);
1420         Cvar_RegisterVariable (&cl_curl_useragent);
1421         Cvar_RegisterVariable (&cl_curl_useragent_append);
1422         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1423         //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1424 }
1425
1426 /*
1427 ====================
1428 Curl_GetDownloadInfo
1429
1430 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1431 The number of elements in the array is returned in int *nDownloads.
1432 const char **additional_info may be set to a string of additional user
1433 information, or to NULL if no such display shall occur. The returned
1434 array must be freed later using Z_Free.
1435 ====================
1436 */
1437 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1438 {
1439         int i;
1440         downloadinfo *di;
1441         Curl_downloadinfo_t *downinfo;
1442
1443         if(!curl_dll)
1444         {
1445                 *nDownloads = 0;
1446                 if(additional_info)
1447                         *additional_info = NULL;
1448                 return NULL;
1449         }
1450
1451         if (curl_mutex) Thread_LockMutex(curl_mutex);
1452
1453         i = 0;
1454         for(di = downloads; di; di = di->next)
1455                 ++i;
1456
1457         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1458         i = 0;
1459         for(di = downloads; di; di = di->next)
1460         {
1461                 // do not show infobars for background downloads
1462                 if(developer.integer <= 0)
1463                         if(di->buffer)
1464                                 continue;
1465                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1466                 if(di->curle)
1467                 {
1468                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1469                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1470                         downinfo[i].queued = false;
1471                 }
1472                 else
1473                 {
1474                         downinfo[i].queued = true;
1475                 }
1476                 ++i;
1477         }
1478
1479         if(additional_info)
1480         {
1481                 // TODO: can I clear command_when_done as soon as the first download fails?
1482                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1483                 {
1484                         if(!strncmp(command_when_done, "connect ", 8))
1485                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1486                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1487                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1488                         else
1489                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1490                         *additional_info = addinfo;
1491                 }
1492                 else
1493                         *additional_info = NULL;
1494         }
1495
1496         *nDownloads = i;
1497         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1498         return downinfo;
1499 }
1500
1501
1502 /*
1503 ====================
1504 Curl_FindPackURL
1505
1506 finds the URL where to find a given package.
1507
1508 For this, it reads a file "curl_urls.txt" of the following format:
1509
1510         data*.pk3       -
1511         revdm*.pk3      http://revdm/downloads/are/here/
1512         *                       http://any/other/stuff/is/here/
1513
1514 The URLs should end in /. If not, downloads will still work, but the cached files
1515 can't be just put into the data directory with the same download configuration
1516 (you might want to do this if you want to tag downloaded files from your
1517 server, but you should not). "-" means "don't download".
1518
1519 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1520 location instead.
1521
1522 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1523 this file for obvious reasons.
1524 ====================
1525 */
1526 static const char *Curl_FindPackURL(const char *filename)
1527 {
1528         static char foundurl[1024]; // invoked only by server
1529         fs_offset_t filesize;
1530         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1531         if(buf && filesize)
1532         {
1533                 // read lines of format "pattern url"
1534                 char *p = buf;
1535                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1536                 qboolean eof = false;
1537
1538                 pattern = p;
1539                 while(!eof)
1540                 {
1541                         switch(*p)
1542                         {
1543                                 case 0:
1544                                         eof = true;
1545                                         // fallthrough
1546                                 case '\n':
1547                                 case '\r':
1548                                         if(pattern && url && patternend)
1549                                         {
1550                                                 if(!urlend)
1551                                                         urlend = p;
1552                                                 *patternend = 0;
1553                                                 *urlend = 0;
1554                                                 if(matchpattern(filename, pattern, true))
1555                                                 {
1556                                                         strlcpy(foundurl, url, sizeof(foundurl));
1557                                                         Z_Free(buf);
1558                                                         return foundurl;
1559                                                 }
1560                                         }
1561                                         pattern = NULL;
1562                                         patternend = NULL;
1563                                         url = NULL;
1564                                         urlend = NULL;
1565                                         break;
1566                                 case ' ':
1567                                 case '\t':
1568                                         if(pattern && !patternend)
1569                                                 patternend = p;
1570                                         else if(url && !urlend)
1571                                                 urlend = p;
1572                                         break;
1573                                 default:
1574                                         if(!pattern)
1575                                                 pattern = p;
1576                                         else if(pattern && patternend && !url)
1577                                                 url = p;
1578                                         break;
1579                         }
1580                         ++p;
1581                 }
1582         }
1583         if(buf)
1584                 Z_Free(buf);
1585         return sv_curl_defaulturl.string;
1586 }
1587
1588 typedef struct requirement_s
1589 {
1590         struct requirement_s *next;
1591         char filename[MAX_OSPATH];
1592 }
1593 requirement;
1594 static requirement *requirements = NULL;
1595
1596
1597 /*
1598 ====================
1599 Curl_RequireFile
1600
1601 Adds the given file to the list of requirements.
1602 ====================
1603 */
1604 void Curl_RequireFile(const char *filename)
1605 {
1606         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1607         req->next = requirements;
1608         strlcpy(req->filename, filename, sizeof(req->filename));
1609         requirements = req;
1610 }
1611
1612 /*
1613 ====================
1614 Curl_ClearRequirements
1615
1616 Clears the list of required files for playing on the current map.
1617 This should be called at every map change.
1618 ====================
1619 */
1620 void Curl_ClearRequirements(void)
1621 {
1622         while(requirements)
1623         {
1624                 requirement *req = requirements;
1625                 requirements = requirements->next;
1626                 Z_Free(req);
1627         }
1628 }
1629
1630 /*
1631 ====================
1632 Curl_SendRequirements
1633
1634 Makes the current host_clients download all files he needs.
1635 This is done by sending him the following console commands:
1636
1637         curl --clear_autodownload
1638         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1639         curl --finish_autodownload
1640 ====================
1641 */
1642 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1643 {
1644         const char *p;
1645         const char *thispack = FS_WhichPack(filename);
1646         const char *packurl;
1647
1648         if(!thispack)
1649                 return false;
1650
1651         p = strrchr(thispack, '/');
1652         if(p)
1653                 thispack = p + 1;
1654
1655         packurl = Curl_FindPackURL(thispack);
1656
1657         if(packurl && *packurl && strcmp(packurl, "-"))
1658         {
1659                 if(!foundone)
1660                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1661
1662                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1663                 strlcat(sendbuffer, thispack, sendbuffer_len);
1664                 if(sv_curl_maxspeed.value > 0)
1665                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1666                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1667                 strlcat(sendbuffer, filename, sendbuffer_len);
1668                 strlcat(sendbuffer, " ", sendbuffer_len);
1669                 strlcat(sendbuffer, packurl, sendbuffer_len);
1670                 strlcat(sendbuffer, thispack, sendbuffer_len);
1671                 strlcat(sendbuffer, "\n", sendbuffer_len);
1672
1673                 return true;
1674         }
1675
1676         return false;
1677 }
1678 void Curl_SendRequirements(void)
1679 {
1680         // for each requirement, find the pack name
1681         char sendbuffer[4096] = "";
1682         requirement *req;
1683         qboolean foundone = false;
1684         const char *p;
1685
1686         for(req = requirements; req; req = req->next)
1687                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1688
1689         p = sv_curl_serverpackages.string;
1690         while(COM_ParseToken_Simple(&p, false, false, true))
1691                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1692
1693         if(foundone)
1694                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1695
1696         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1697                 Host_ClientCommands("%s", sendbuffer);
1698         else
1699                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1700 }