]> de.git.xonotic.org Git - xonotic/darkplaces.git/blob - libcurl.c
7c9b7dbdf0060802087ef8769881a7d686a8ddbc
[xonotic/darkplaces.git] / libcurl.c
1 #include "quakedef.h"
2 #include "fs.h"
3 #include "libcurl.h"
4
5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
9 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
10
11 /*
12 =================================================================
13
14   Minimal set of definitions from libcurl
15
16   WARNING: for a matter of simplicity, several pointer types are
17   casted to "void*", and most enumerated values are not included
18
19 =================================================================
20 */
21
22 typedef struct CURL_s CURL;
23 typedef struct CURLM_s CURLM;
24 typedef enum
25 {
26         CURLE_OK = 0
27 }
28 CURLcode;
29 typedef enum
30 {
31         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
32         CURLM_OK = 0
33 }
34 CURLMcode;
35 #define CURL_GLOBAL_NOTHING 0
36 #define CURL_GLOBAL_SSL 1
37 #define CURL_GLOBAL_WIN32 2
38 #define CURLOPTTYPE_LONG          0
39 #define CURLOPTTYPE_OBJECTPOINT   10000
40 #define CURLOPTTYPE_FUNCTIONPOINT 20000
41 #define CURLOPTTYPE_OFF_T         30000
42 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
43 typedef enum
44 {
45         CINIT(WRITEDATA, OBJECTPOINT, 1),
46         CINIT(URL,  OBJECTPOINT, 2),
47         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
48         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
49         CINIT(REFERER, OBJECTPOINT, 16),
50         CINIT(USERAGENT, OBJECTPOINT, 18),
51         CINIT(RESUME_FROM, LONG, 21),
52         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
53         CINIT(PRIVATE, OBJECTPOINT, 103),
54         CINIT(LOW_SPEED_LIMIT, LONG , 19),
55         CINIT(LOW_SPEED_TIME, LONG, 20),
56 }
57 CURLoption;
58 typedef enum
59 {
60         CURLINFO_TEXT = 0,
61         CURLINFO_HEADER_IN,    /* 1 */
62         CURLINFO_HEADER_OUT,   /* 2 */
63         CURLINFO_DATA_IN,      /* 3 */
64         CURLINFO_DATA_OUT,     /* 4 */
65         CURLINFO_SSL_DATA_IN,  /* 5 */
66         CURLINFO_SSL_DATA_OUT, /* 6 */
67         CURLINFO_END
68 }
69 curl_infotype;
70 #define CURLINFO_STRING   0x100000
71 #define CURLINFO_LONG     0x200000
72 #define CURLINFO_DOUBLE   0x300000
73 #define CURLINFO_SLIST    0x400000
74 #define CURLINFO_MASK     0x0fffff
75 #define CURLINFO_TYPEMASK 0xf00000
76 typedef enum
77 {
78         CURLINFO_NONE, /* first, never use this */
79         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
80         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
81         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
82         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
83         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
84         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
85         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
86         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
87         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
88         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
89         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
90         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
91         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
92         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
93         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
94         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
95         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
96         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
97         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
98         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
99         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
100         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
101         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
102         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
103         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
104         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
105         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
106 }
107 CURLINFO;
108
109 typedef enum
110 {
111         CURLMSG_NONE, /* first, not used */
112         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
113                                          the CURLcode of the transfer */
114         CURLMSG_LAST
115 }
116 CURLMSG;
117 typedef struct
118 {
119         CURLMSG msg;       /* what this message means */
120         CURL *easy_handle; /* the handle it concerns */
121         union
122         {
123                 void *whatever;    /* message-specific data */
124                 CURLcode result;   /* return code for transfer */
125         }
126         data;
127 }
128 CURLMsg;
129
130 static void (*qcurl_global_init) (long flags);
131 static void (*qcurl_global_cleanup) ();
132
133 static CURL * (*qcurl_easy_init) ();
134 static void (*qcurl_easy_cleanup) (CURL *handle);
135 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
136 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
137 static const char * (*qcurl_easy_strerror) (CURLcode);
138
139 static CURLM * (*qcurl_multi_init) ();
140 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
141 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
142 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
143 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
144 static void (*qcurl_multi_cleanup) (CURLM *);
145 static const char * (*qcurl_multi_strerror) (CURLcode);
146
147 static dllfunction_t curlfuncs[] =
148 {
149         {"curl_global_init",            (void **) &qcurl_global_init},
150         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
151         {"curl_easy_init",                      (void **) &qcurl_easy_init},
152         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
153         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
154         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
155         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
156         {"curl_multi_init",                     (void **) &qcurl_multi_init},
157         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
158         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
159         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
160         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
161         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
162         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
163         {NULL, NULL}
164 };
165
166 // Handle for CURL DLL
167 static dllhandle_t curl_dll = NULL;
168 // will be checked at many places to find out if qcurl calls are allowed
169
170 typedef struct downloadinfo_s
171 {
172         char filename[MAX_QPATH];
173         char url[256];
174         char referer[256];
175         qfile_t *stream;
176         fs_offset_t startpos;
177         CURL *curle;
178         qboolean started;
179         qboolean ispak;
180         unsigned long bytes_received;
181         struct downloadinfo_s *next, *prev;
182         qboolean forthismap;
183
184         unsigned char *buffer;
185         size_t buffersize;
186         curl_callback_t callback;
187         void *callback_data;
188 }
189 downloadinfo;
190 static downloadinfo *downloads = NULL;
191 static int numdownloads = 0;
192
193 static qboolean noclear = FALSE;
194
195 static int numdownloads_fail = 0;
196 static int numdownloads_success = 0;
197 static int numdownloads_added = 0;
198 static char command_when_done[256] = "";
199 static char command_when_error[256] = "";
200
201 /*
202 ====================
203 Curl_CommandWhenDone
204
205 Sets the command which is to be executed when the last download completes AND
206 all downloads since last server connect ended with a successful status.
207 Setting the command to NULL clears it.
208 ====================
209 */
210 void Curl_CommandWhenDone(const char *cmd)
211 {
212         if(!curl_dll)
213                 return;
214         if(cmd)
215                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
216         else
217                 *command_when_done = 0;
218 }
219
220 /*
221 FIXME
222 Do not use yet. Not complete.
223 Problem: what counts as an error?
224 */
225
226 void Curl_CommandWhenError(const char *cmd)
227 {
228         if(!curl_dll)
229                 return;
230         if(cmd)
231                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
232         else
233                 *command_when_error = 0;
234 }
235
236 /*
237 ====================
238 Curl_Clear_forthismap
239
240 Clears the "will disconnect on failure" flags.
241 ====================
242 */
243 void Curl_Clear_forthismap()
244 {
245         downloadinfo *di;
246         if(noclear)
247                 return;
248         for(di = downloads; di; di = di->next)
249                 di->forthismap = false;
250         Curl_CommandWhenError(NULL);
251         Curl_CommandWhenDone(NULL);
252         numdownloads_fail = 0;
253         numdownloads_success = 0;
254         numdownloads_added = 0;
255 }
256
257 /*
258 ====================
259 Curl_Have_forthismap
260
261 Returns true if a download needed for the current game is running.
262 ====================
263 */
264 qboolean Curl_Have_forthismap()
265 {
266         return numdownloads_added;
267 }
268
269 void Curl_Register_predownload()
270 {
271         Curl_CommandWhenDone("cl_begindownloads");
272         Curl_CommandWhenError("cl_begindownloads");
273 }
274
275 /*
276 ====================
277 Curl_CheckCommandWhenDone
278
279 Checks if a "done command" is to be executed.
280 All downloads finished, at least one success since connect, no single failure
281 -> execute the command.
282 */
283 static void Curl_CheckCommandWhenDone()
284 {
285         if(!curl_dll)
286                 return;
287         if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
288         {
289                 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
290                 Cbuf_AddText("\n");
291                 Cbuf_AddText(command_when_done);
292                 Cbuf_AddText("\n");
293                 Curl_Clear_forthismap();
294         }
295         else if(numdownloads_added && numdownloads_fail && *command_when_error)
296         {
297                 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
298                 Cbuf_AddText("\n");
299                 Cbuf_AddText(command_when_error);
300                 Cbuf_AddText("\n");
301                 Curl_Clear_forthismap();
302         }
303 }
304
305 /*
306 ====================
307 CURL_CloseLibrary
308
309 Load the cURL DLL
310 ====================
311 */
312 static qboolean CURL_OpenLibrary (void)
313 {
314         const char* dllnames [] =
315         {
316 #if defined(WIN64)
317                 "libcurl64.dll",
318 #elif defined(WIN32)
319                 "libcurl-4.dll",
320                 "libcurl-3.dll",
321 #elif defined(MACOSX)
322                 "libcurl.4.dylib", // Mac OS X Notyetreleased
323                 "libcurl.3.dylib", // Mac OS X Tiger
324                 "libcurl.2.dylib", // Mac OS X Panther
325 #else
326                 "libcurl.so.4",
327                 "libcurl.so.3",
328                 "libcurl.so", // FreeBSD
329 #endif
330                 NULL
331         };
332
333         // Already loaded?
334         if (curl_dll)
335                 return true;
336
337         // Load the DLL
338         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
339 }
340
341
342 /*
343 ====================
344 CURL_CloseLibrary
345
346 Unload the cURL DLL
347 ====================
348 */
349 static void CURL_CloseLibrary (void)
350 {
351         Sys_UnloadLibrary (&curl_dll);
352 }
353
354
355 static CURLM *curlm = NULL;
356 static unsigned long bytes_received = 0; // used for bandwidth throttling
357 static double curltime = 0;
358
359 /*
360 ====================
361 CURL_fwrite
362
363 fwrite-compatible function that writes the data to a file. libcurl can call
364 this.
365 ====================
366 */
367 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
368 {
369         fs_offset_t ret = -1;
370         size_t bytes = size * nmemb;
371         downloadinfo *di = (downloadinfo *) vdi;
372
373         if(di->buffer)
374         {
375                 if(di->bytes_received + bytes <= di->buffersize)
376                 {
377                         memcpy(di->buffer + di->bytes_received, data, bytes);
378                         ret = bytes;
379                 }
380                 // otherwise: buffer overrun, ret stays -1
381         }
382
383         if(di->stream)
384         {
385                 ret = FS_Write(di->stream, data, bytes);
386         }
387
388         bytes_received += bytes;
389         di->bytes_received += bytes;
390
391         return ret; // why not ret / nmemb?
392 }
393
394 typedef enum
395 {
396         CURL_DOWNLOAD_SUCCESS = 0,
397         CURL_DOWNLOAD_FAILED,
398         CURL_DOWNLOAD_ABORTED,
399         CURL_DOWNLOAD_SERVERERROR
400 }
401 CurlStatus;
402
403 /*
404 ====================
405 Curl_EndDownload
406
407 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
408 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
409 code from libcurl, or 0, if another error has occurred.
410 ====================
411 */
412 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
413 {
414         qboolean ok = false;
415         if(!curl_dll)
416                 return;
417         switch(status)
418         {
419                 case CURL_DOWNLOAD_SUCCESS:
420                         Con_Printf("Download of %s: OK\n", di->filename);
421                         ok = true;
422
423                         if(di->callback)
424                                 di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
425                         break;
426                 case CURL_DOWNLOAD_FAILED:
427                         Con_Printf("Download of %s: FAILED\n", di->filename);
428                         if(error)
429                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
430
431                         if(di->callback)
432                                 di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
433                         break;
434                 case CURL_DOWNLOAD_ABORTED:
435                         Con_Printf("Download of %s: ABORTED\n", di->filename);
436
437                         if(di->callback)
438                                 di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
439                         break;
440                 case CURL_DOWNLOAD_SERVERERROR:
441                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
442
443                         // reopen to enforce it to have zero bytes again
444                         if(di->stream)
445                         {
446                                 FS_Close(di->stream);
447                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
448                         }
449
450                         if(di->callback)
451                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
452                         break;
453                 default:
454                         Con_Printf("Download of %s: ???\n", di->filename);
455
456                         if(di->callback)
457                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
458                         break;
459         }
460
461         if(di->curle)
462         {
463                 qcurl_multi_remove_handle(curlm, di->curle);
464                 qcurl_easy_cleanup(di->curle);
465         }
466
467         if(ok && !di->bytes_received)
468         {
469                 Con_Printf("ERROR: empty file\n");
470                 ok = false;
471         }
472
473         if(di->stream)
474                 FS_Close(di->stream);
475
476         if(ok && di->ispak)
477                 ok = FS_AddPack(di->filename, NULL, true);
478
479         if(di->prev)
480                 di->prev->next = di->next;
481         else
482                 downloads = di->next;
483         if(di->next)
484                 di->next->prev = di->prev;
485
486         --numdownloads;
487         if(di->forthismap)
488         {
489                 if(ok)
490                         ++numdownloads_success;
491                 else
492                         ++numdownloads_fail;
493         }
494         Z_Free(di);
495 }
496
497 /*
498 ====================
499 CheckPendingDownloads
500
501 checks if there are free download slots to start new downloads in.
502 To not start too many downloads at once, only one download is added at a time,
503 up to a maximum number of cl_curl_maxdownloads are running.
504 ====================
505 */
506 static void CheckPendingDownloads()
507 {
508         if(!curl_dll)
509                 return;
510         if(numdownloads < cl_curl_maxdownloads.integer)
511         {
512                 downloadinfo *di;
513                 for(di = downloads; di; di = di->next)
514                 {
515                         if(!di->started)
516                         {
517                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
518
519                                 if(!di->buffer)
520                                 {
521                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
522                                         if(!di->stream)
523                                         {
524                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
525                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
526                                                 return;
527                                         }
528                                         FS_Seek(di->stream, 0, SEEK_END);
529                                         di->startpos = FS_Tell(di->stream);
530                                 }
531                                 else
532                                 {
533                                         di->startpos = 0;
534                                 }
535
536                                 if(di->startpos > 0)
537                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
538                                 Con_Print("...\n");
539
540                                 di->curle = qcurl_easy_init();
541                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
542                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
543                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
544                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
545                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
546                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
547                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
548                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
549                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
550                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
551                                 qcurl_multi_add_handle(curlm, di->curle);
552                                 di->started = true;
553                                 ++numdownloads;
554                                 if(numdownloads >= cl_curl_maxdownloads.integer)
555                                         break;
556                         }
557                 }
558         }
559 }
560
561 /*
562 ====================
563 Curl_Init
564
565 this function MUST be called before using anything else in this file.
566 On Win32, this must be called AFTER WSAStartup has been done!
567 ====================
568 */
569 void Curl_Init()
570 {
571         CURL_OpenLibrary();
572         if(!curl_dll)
573                 return;
574         qcurl_global_init(CURL_GLOBAL_NOTHING);
575         curlm = qcurl_multi_init();
576 }
577
578 /*
579 ====================
580 Curl_Shutdown
581
582 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
583 ====================
584 */
585 void Curl_ClearRequirements();
586 void Curl_Shutdown()
587 {
588         if(!curl_dll)
589                 return;
590         Curl_ClearRequirements();
591         Curl_CancelAll();
592         CURL_CloseLibrary();
593         curl_dll = NULL;
594 }
595
596 /*
597 ====================
598 Curl_Find
599
600 Finds the internal information block for a download given by file name.
601 ====================
602 */
603 static downloadinfo *Curl_Find(const char *filename)
604 {
605         downloadinfo *di;
606         if(!curl_dll)
607                 return NULL;
608         for(di = downloads; di; di = di->next)
609                 if(!strcasecmp(di->filename, filename))
610                         return di;
611         return NULL;
612 }
613
614 /*
615 ====================
616 Curl_Begin
617
618 Starts a download of a given URL to the file name portion of this URL (or name
619 if given) in the "dlcache/" folder.
620 ====================
621 */
622 static qboolean Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
623 {
624         if(!curl_dll)
625         {
626                 return false;
627         }
628         else
629         {
630                 char fn[MAX_QPATH];
631                 const char *p, *q;
632                 size_t length;
633                 downloadinfo *di;
634
635                 // Note: This extraction of the file name portion is NOT entirely correct.
636                 //
637                 // It does the following:
638                 //
639                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
640                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
641                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
642                 //
643                 // However, I'd like to keep this "buggy" behavior so that PHP script
644                 // authors can write download scripts without having to enable
645                 // AcceptPathInfo on Apache. They just have to ensure that their script
646                 // can be called with such a "fake" path name like
647                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
648                 //
649                 // By the way, such PHP scripts should either send the file or a
650                 // "Location:" redirect; PHP code example:
651                 //
652                 //   header("Location: http://www.example.com/");
653                 //
654                 // By the way, this will set User-Agent to something like
655                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
656                 // dp://serverhost:serverport/ so you can filter on this; an example
657                 // httpd log file line might be:
658                 //
659                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
660
661                 if(!name)
662                         name = URL;
663
664                 if(!buf)
665                 {
666                         p = strrchr(name, '/');
667                         p = p ? (p+1) : name;
668                         q = strchr(p, '?');
669                         length = q ? (size_t)(q - p) : strlen(p);
670                         dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
671
672                         name = fn; // make it point back
673
674                         // already downloading the file?
675                         {
676                                 downloadinfo *di = Curl_Find(fn);
677                                 if(di)
678                                 {
679                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
680
681                                         // however, if it was not for this map yet...
682                                         if(forthismap && !di->forthismap)
683                                         {
684                                                 di->forthismap = true;
685                                                 // this "fakes" a download attempt so the client will wait for
686                                                 // the download to finish and then reconnect
687                                                 ++numdownloads_added;
688                                         }
689
690                                         return false;
691                                 }
692                         }
693
694                         if(ispak && FS_FileExists(fn))
695                         {
696                                 qboolean already_loaded;
697                                 if(FS_AddPack(fn, &already_loaded, true))
698                                 {
699                                         Con_DPrintf("%s already exists, not downloading!\n", fn);
700                                         if(already_loaded)
701                                                 Con_DPrintf("(pak was already loaded)\n");
702                                         else
703                                         {
704                                                 if(forthismap)
705                                                 {
706                                                         ++numdownloads_added;
707                                                         ++numdownloads_success;
708                                                 }
709                                         }
710
711                                         return false;
712                                 }
713                                 else
714                                 {
715                                         qfile_t *f = FS_OpenVirtualFile(fn, false);
716                                         if(f)
717                                         {
718                                                 char buf[4] = {0};
719                                                 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
720
721                                                 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
722                                                 {
723                                                         Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
724                                                         FS_Close(f);
725                                                         f = FS_OpenRealFile(fn, "wb", false);
726                                                         if(f)
727                                                                 FS_Close(f);
728                                                 }
729                                                 else
730                                                 {
731                                                         // OK
732                                                         FS_Close(f);
733                                                 }
734                                         }
735                                 }
736                         }
737                 }
738
739                 if(forthismap)
740                         ++numdownloads_added;
741                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
742                 strlcpy(di->filename, name, sizeof(di->filename));
743                 strlcpy(di->url, URL, sizeof(di->url));
744                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
745                 di->forthismap = forthismap;
746                 di->stream = NULL;
747                 di->startpos = 0;
748                 di->curle = NULL;
749                 di->started = false;
750                 di->ispak = ispak;
751                 di->bytes_received = 0;
752                 di->next = downloads;
753                 di->prev = NULL;
754                 if(di->next)
755                         di->next->prev = di;
756
757                 di->buffer = buf;
758                 di->buffersize = bufsize;
759                 di->callback = callback;
760                 di->callback_data = cbdata;
761
762                 downloads = di;
763                 return true;
764         }
765 }
766
767 qboolean Curl_Begin_ToFile(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
768 {
769         return Curl_Begin(URL, name, ispak, forthismap, NULL, 0, NULL, NULL);
770 }
771 qboolean Curl_Begin_ToMemory(const char *URL, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
772 {
773         return Curl_Begin(URL, NULL, false, false, buf, bufsize, callback, cbdata);
774 }
775
776 /*
777 ====================
778 Curl_Run
779
780 call this regularily as this will always download as much as possible without
781 blocking.
782 ====================
783 */
784 void Curl_Run()
785 {
786         noclear = FALSE;
787
788         if(!cl_curl_enabled.integer)
789                 return;
790
791         if(!curl_dll)
792                 return;
793
794         Curl_CheckCommandWhenDone();
795
796         if(!downloads)
797                 return;
798
799         if(realtime < curltime) // throttle
800                 return;
801
802         {
803                 int remaining;
804                 CURLMcode mc;
805
806                 do
807                 {
808                         mc = qcurl_multi_perform(curlm, &remaining);
809                 }
810                 while(mc == CURLM_CALL_MULTI_PERFORM);
811
812                 for(;;)
813                 {
814                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
815                         if(!msg)
816                                 break;
817                         if(msg->msg == CURLMSG_DONE)
818                         {
819                                 downloadinfo *di;
820                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
821                                 CURLcode result;
822                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
823                                 result = msg->data.result;
824                                 if(result)
825                                 {
826                                         failed = CURL_DOWNLOAD_FAILED;
827                                 }
828                                 else
829                                 {
830                                         long code;
831                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
832                                         switch(code / 100)
833                                         {
834                                                 case 4: // e.g. 404?
835                                                 case 5: // e.g. 500?
836                                                         failed = CURL_DOWNLOAD_SERVERERROR;
837                                                         result = code;
838                                                         break;
839                                         }
840                                 }
841
842                                 Curl_EndDownload(di, failed, result);
843                         }
844                 }
845         }
846
847         CheckPendingDownloads();
848
849         // when will we curl the next time?
850         // we will wait a bit to ensure our download rate is kept.
851         // we now know that realtime >= curltime... so set up a new curltime
852         if(cl_curl_maxspeed.value > 0)
853         {
854                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
855                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
856                 bytes_received -= bytes;
857         }
858         else
859                 curltime = realtime;
860 }
861
862 /*
863 ====================
864 Curl_CancelAll
865
866 Stops ALL downloads.
867 ====================
868 */
869 void Curl_CancelAll()
870 {
871         if(!curl_dll)
872                 return;
873
874         while(downloads)
875         {
876                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
877                 // INVARIANT: downloads will point to the next download after that!
878         }
879 }
880
881 /*
882 ====================
883 Curl_Running
884
885 returns true iff there is a download running.
886 ====================
887 */
888 qboolean Curl_Running()
889 {
890         if(!curl_dll)
891                 return false;
892
893         return downloads != NULL;
894 }
895
896 /*
897 ====================
898 Curl_GetDownloadAmount
899
900 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
901 for the given download.
902 ====================
903 */
904 static double Curl_GetDownloadAmount(downloadinfo *di)
905 {
906         if(!curl_dll)
907                 return -2;
908         if(di->curle)
909         {
910                 double length;
911                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
912                 if(length > 0)
913                         return di->bytes_received / length;
914                 else
915                         return 0;
916         }
917         else
918                 return -1;
919 }
920
921 /*
922 ====================
923 Curl_GetDownloadSpeed
924
925 returns the speed of the given download in bytes per second
926 ====================
927 */
928 static double Curl_GetDownloadSpeed(downloadinfo *di)
929 {
930         if(!curl_dll)
931                 return -2;
932         if(di->curle)
933         {
934                 double speed;
935                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
936                 return speed;
937         }
938         else
939                 return -1;
940 }
941
942 /*
943 ====================
944 Curl_Info_f
945
946 prints the download list
947 ====================
948 */
949 // TODO rewrite using Curl_GetDownloadInfo?
950 static void Curl_Info_f()
951 {
952         downloadinfo *di;
953         if(!curl_dll)
954                 return;
955         if(Curl_Running())
956         {
957                 Con_Print("Currently running downloads:\n");
958                 for(di = downloads; di; di = di->next)
959                 {
960                         double speed, percent;
961                         Con_Printf("  %s -> %s ",  di->url, di->filename);
962                         percent = 100.0 * Curl_GetDownloadAmount(di);
963                         speed = Curl_GetDownloadSpeed(di);
964                         if(percent >= 0)
965                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
966                         else
967                                 Con_Print("(queued)\n");
968                 }
969         }
970         else
971         {
972                 Con_Print("No downloads running.\n");
973         }
974 }
975
976 /*
977 ====================
978 Curl_Curl_f
979
980 implements the "curl" console command
981
982 curl --info
983 curl --cancel
984 curl --cancel filename
985 curl url
986
987 For internal use:
988
989 curl [--pak] [--forthismap] [--for filename filename...] url
990         --pak: after downloading, load the package into the virtual file system
991         --for filename...: only download of at least one of the named files is missing
992         --forthismap: don't reconnect on failure
993
994 curl --clear_autodownload
995         clears the download success/failure counters
996
997 curl --finish_autodownload
998         if at least one download has been started, disconnect and drop to the menu
999         once the last download completes successfully, reconnect to the current server
1000 ====================
1001 */
1002 void Curl_Curl_f(void)
1003 {
1004         int i;
1005         int end;
1006         qboolean pak = false;
1007         qboolean forthismap = false;
1008         const char *url;
1009         const char *name = 0;
1010
1011         if(!curl_dll)
1012         {
1013                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1014                 return;
1015         }
1016
1017         if(!cl_curl_enabled.integer)
1018         {
1019                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1020                 return;
1021         }
1022
1023         for(i = 0; i != Cmd_Argc(); ++i)
1024                 Con_DPrintf("%s ", Cmd_Argv(i));
1025         Con_DPrint("\n");
1026
1027         if(Cmd_Argc() < 2)
1028         {
1029                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1030                 return;
1031         }
1032
1033         url = Cmd_Argv(Cmd_Argc() - 1);
1034         end = Cmd_Argc();
1035
1036         for(i = 1; i != end; ++i)
1037         {
1038                 const char *a = Cmd_Argv(i);
1039                 if(!strcmp(a, "--info"))
1040                 {
1041                         Curl_Info_f();
1042                         return;
1043                 }
1044                 else if(!strcmp(a, "--cancel"))
1045                 {
1046                         if(i == end - 1) // last argument
1047                                 Curl_CancelAll();
1048                         else
1049                         {
1050                                 downloadinfo *di = Curl_Find(url);
1051                                 if(di)
1052                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1053                                 else
1054                                         Con_Print("download not found\n");
1055                         }
1056                         return;
1057                 }
1058                 else if(!strcmp(a, "--pak"))
1059                 {
1060                         pak = true;
1061                 }
1062                 else if(!strcmp(a, "--for"))
1063                 {
1064                         for(i = i + 1; i != end - 1; ++i)
1065                         {
1066                                 if(!FS_FileExists(Cmd_Argv(i)))
1067                                         goto needthefile; // why can't I have a "double break"?
1068                         }
1069                         // if we get here, we have all the files...
1070                         return;
1071                 }
1072                 else if(!strcmp(a, "--forthismap"))
1073                 {
1074                         forthismap = true;
1075                 }
1076                 else if(!strcmp(a, "--as"))
1077                 {
1078                         if(i < end - 1)
1079                         {
1080                                 ++i;
1081                                 name = Cmd_Argv(i);
1082                         }
1083                 }
1084                 else if(!strcmp(a, "--clear_autodownload"))
1085                 {
1086                         // mark all running downloads as "not for this map", so if they
1087                         // fail, it does not matter
1088                         Curl_Clear_forthismap();
1089                         return;
1090                 }
1091                 else if(!strcmp(a, "--finish_autodownload"))
1092                 {
1093                         if(numdownloads_added)
1094                         {
1095                                 char donecommand[256];
1096                                 if(cls.netcon)
1097                                 {
1098                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1099                                         {
1100                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1101                                                 Curl_CommandWhenDone(donecommand);
1102                                                 noclear = TRUE;
1103                                                 CL_Disconnect();
1104                                                 noclear = FALSE;
1105                                                 Curl_CheckCommandWhenDone();
1106                                         }
1107                                         else
1108                                                 Curl_Register_predownload();
1109                                 }
1110                         }
1111                         return;
1112                 }
1113                 else if(*a == '-')
1114                 {
1115                         Con_Printf("invalid option %s\n", a);
1116                         return;
1117                 }
1118         }
1119
1120 needthefile:
1121         Curl_Begin_ToFile(url, name, pak, forthismap);
1122 }
1123
1124 /*
1125 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1126 {
1127         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1128         Z_Free(buffer);
1129 }
1130
1131 void Curl_CurlCat_f(void)
1132 {
1133         unsigned char *buf;
1134         const char *url = Cmd_Argv(1);
1135         buf = Z_Malloc(16384);
1136         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1137 }
1138 */
1139
1140 /*
1141 ====================
1142 Curl_Init_Commands
1143
1144 loads the commands and cvars this library uses
1145 ====================
1146 */
1147 void Curl_Init_Commands(void)
1148 {
1149         Cvar_RegisterVariable (&cl_curl_enabled);
1150         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1151         Cvar_RegisterVariable (&cl_curl_maxspeed);
1152         Cvar_RegisterVariable (&sv_curl_defaulturl);
1153         Cvar_RegisterVariable (&sv_curl_serverpackages);
1154         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1155         //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1156 }
1157
1158 /*
1159 ====================
1160 Curl_GetDownloadInfo
1161
1162 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1163 The number of elements in the array is returned in int *nDownloads.
1164 const char **additional_info may be set to a string of additional user
1165 information, or to NULL if no such display shall occur. The returned
1166 array must be freed later using Z_Free.
1167 ====================
1168 */
1169 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1170 {
1171         int n, i;
1172         downloadinfo *di;
1173         Curl_downloadinfo_t *downinfo;
1174         static char addinfo[128];
1175
1176         if(!curl_dll)
1177         {
1178                 *nDownloads = 0;
1179                 if(additional_info)
1180                         *additional_info = NULL;
1181                 return NULL;
1182         }
1183
1184         n = 0;
1185         for(di = downloads; di; di = di->next)
1186                 ++n;
1187
1188         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
1189         i = 0;
1190         for(di = downloads; di; di = di->next)
1191         {
1192                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1193                 if(di->curle)
1194                 {
1195                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1196                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1197                         downinfo[i].queued = false;
1198                 }
1199                 else
1200                 {
1201                         downinfo[i].queued = true;
1202                 }
1203                 ++i;
1204         }
1205
1206         if(additional_info)
1207         {
1208                 // TODO: can I clear command_when_done as soon as the first download fails?
1209                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1210                 {
1211                         if(!strncmp(command_when_done, "connect ", 8))
1212                                 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1213                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1214                                 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1215                         else
1216                                 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1217                         *additional_info = addinfo;
1218                 }
1219                 else
1220                         *additional_info = NULL;
1221         }
1222
1223         *nDownloads = n;
1224         return downinfo;
1225 }
1226
1227
1228 /*
1229 ====================
1230 Curl_FindPackURL
1231
1232 finds the URL where to find a given package.
1233
1234 For this, it reads a file "curl_urls.txt" of the following format:
1235
1236         data*.pk3       -
1237         revdm*.pk3      http://revdm/downloads/are/here/
1238         *                       http://any/other/stuff/is/here/
1239
1240 The URLs should end in /. If not, downloads will still work, but the cached files
1241 can't be just put into the data directory with the same download configuration
1242 (you might want to do this if you want to tag downloaded files from your
1243 server, but you should not). "-" means "don't download".
1244
1245 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1246 location instead.
1247
1248 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1249 this file for obvious reasons.
1250 ====================
1251 */
1252 static const char *Curl_FindPackURL(const char *filename)
1253 {
1254         static char foundurl[256];
1255         fs_offset_t filesize;
1256         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1257         if(buf && filesize)
1258         {
1259                 // read lines of format "pattern url"
1260                 char *p = buf;
1261                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1262                 qboolean eof = false;
1263
1264                 pattern = p;
1265                 while(!eof)
1266                 {
1267                         switch(*p)
1268                         {
1269                                 case 0:
1270                                         eof = true;
1271                                         // fallthrough
1272                                 case '\n':
1273                                 case '\r':
1274                                         if(pattern && url && patternend)
1275                                         {
1276                                                 if(!urlend)
1277                                                         urlend = p;
1278                                                 *patternend = 0;
1279                                                 *urlend = 0;
1280                                                 if(matchpattern(filename, pattern, true))
1281                                                 {
1282                                                         strlcpy(foundurl, url, sizeof(foundurl));
1283                                                         Z_Free(buf);
1284                                                         return foundurl;
1285                                                 }
1286                                         }
1287                                         pattern = NULL;
1288                                         patternend = NULL;
1289                                         url = NULL;
1290                                         urlend = NULL;
1291                                         break;
1292                                 case ' ':
1293                                 case '\t':
1294                                         if(pattern && !patternend)
1295                                                 patternend = p;
1296                                         else if(url && !urlend)
1297                                                 urlend = p;
1298                                         break;
1299                                 default:
1300                                         if(!pattern)
1301                                                 pattern = p;
1302                                         else if(pattern && patternend && !url)
1303                                                 url = p;
1304                                         break;
1305                         }
1306                         ++p;
1307                 }
1308         }
1309         if(buf)
1310                 Z_Free(buf);
1311         return sv_curl_defaulturl.string;
1312 }
1313
1314 typedef struct requirement_s
1315 {
1316         struct requirement_s *next;
1317         char filename[MAX_QPATH];
1318 }
1319 requirement;
1320 static requirement *requirements = NULL;
1321
1322
1323 /*
1324 ====================
1325 Curl_RequireFile
1326
1327 Adds the given file to the list of requirements.
1328 ====================
1329 */
1330 void Curl_RequireFile(const char *filename)
1331 {
1332         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1333         req->next = requirements;
1334         strlcpy(req->filename, filename, sizeof(req->filename));
1335         requirements = req;
1336 }
1337
1338 /*
1339 ====================
1340 Curl_ClearRequirements
1341
1342 Clears the list of required files for playing on the current map.
1343 This should be called at every map change.
1344 ====================
1345 */
1346 void Curl_ClearRequirements()
1347 {
1348         const char *p;
1349         while(requirements)
1350         {
1351                 requirement *req = requirements;
1352                 requirements = requirements->next;
1353                 Z_Free(req);
1354         }
1355         p = sv_curl_serverpackages.string;
1356         Con_DPrintf("Require all of: %s\n", p);
1357         while(COM_ParseToken_Simple(&p, false, false))
1358         {
1359                 Con_DPrintf("Require: %s\n", com_token);
1360                 Curl_RequireFile(com_token);
1361         }
1362 }
1363
1364 /*
1365 ====================
1366 Curl_SendRequirements
1367
1368 Makes the current host_clients download all files he needs.
1369 This is done by sending him the following console commands:
1370
1371         curl --clear_autodownload
1372         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1373         curl --finish_autodownload
1374 ====================
1375 */
1376 void Curl_SendRequirements()
1377 {
1378         // for each requirement, find the pack name
1379         char sendbuffer[4096] = "";
1380         requirement *req;
1381         qboolean foundone = false;
1382
1383         for(req = requirements; req; req = req->next)
1384         {
1385                 const char *p;
1386                 const char *thispack = FS_WhichPack(req->filename);
1387                 const char *packurl;
1388
1389                 if(!thispack)
1390                         continue;
1391
1392                 p = strrchr(thispack, '/');
1393                 if(p)
1394                         thispack = p + 1;
1395
1396                 packurl = Curl_FindPackURL(thispack);
1397
1398                 if(packurl && *packurl && strcmp(packurl, "-"))
1399                 {
1400                         if(!foundone)
1401                                 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1402
1403                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1404                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1405                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1406                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1407                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1408                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1409                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1410                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1411
1412                         foundone = true;
1413                 }
1414         }
1415
1416         if(foundone)
1417                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1418
1419         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1420                 Host_ClientCommands("%s", sendbuffer);
1421         else
1422                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1423 }