5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
9 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
10 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
13 =================================================================
15 Minimal set of definitions from libcurl
17 WARNING: for a matter of simplicity, several pointer types are
18 casted to "void*", and most enumerated values are not included
20 =================================================================
23 typedef struct CURL_s CURL;
24 typedef struct CURLM_s CURLM;
25 typedef struct curl_slist curl_slist;
33 CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
37 #define CURL_GLOBAL_NOTHING 0
38 #define CURL_GLOBAL_SSL 1
39 #define CURL_GLOBAL_WIN32 2
40 #define CURLOPTTYPE_LONG 0
41 #define CURLOPTTYPE_OBJECTPOINT 10000
42 #define CURLOPTTYPE_FUNCTIONPOINT 20000
43 #define CURLOPTTYPE_OFF_T 30000
44 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
47 CINIT(WRITEDATA, OBJECTPOINT, 1),
48 CINIT(URL, OBJECTPOINT, 2),
49 CINIT(ERRORBUFFER, OBJECTPOINT, 10),
50 CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
51 CINIT(POSTFIELDS, OBJECTPOINT, 15),
52 CINIT(REFERER, OBJECTPOINT, 16),
53 CINIT(USERAGENT, OBJECTPOINT, 18),
54 CINIT(LOW_SPEED_LIMIT, LONG , 19),
55 CINIT(LOW_SPEED_TIME, LONG, 20),
56 CINIT(RESUME_FROM, LONG, 21),
57 CINIT(HTTPHEADER, OBJECTPOINT, 23),
58 CINIT(POST, LONG, 47), /* HTTP POST method */
59 CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */
60 CINIT(POSTFIELDSIZE, LONG, 60),
61 CINIT(PRIVATE, OBJECTPOINT, 103),
62 CINIT(PROTOCOLS, LONG, 181),
63 CINIT(REDIR_PROTOCOLS, LONG, 182)
66 #define CURLPROTO_HTTP (1<<0)
67 #define CURLPROTO_HTTPS (1<<1)
68 #define CURLPROTO_FTP (1<<2)
72 CURLINFO_HEADER_IN, /* 1 */
73 CURLINFO_HEADER_OUT, /* 2 */
74 CURLINFO_DATA_IN, /* 3 */
75 CURLINFO_DATA_OUT, /* 4 */
76 CURLINFO_SSL_DATA_IN, /* 5 */
77 CURLINFO_SSL_DATA_OUT, /* 6 */
81 #define CURLINFO_STRING 0x100000
82 #define CURLINFO_LONG 0x200000
83 #define CURLINFO_DOUBLE 0x300000
84 #define CURLINFO_SLIST 0x400000
85 #define CURLINFO_MASK 0x0fffff
86 #define CURLINFO_TYPEMASK 0xf00000
89 CURLINFO_NONE, /* first, never use this */
90 CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1,
91 CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2,
92 CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3,
93 CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4,
94 CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5,
95 CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
96 CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7,
97 CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8,
98 CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9,
99 CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10,
100 CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11,
101 CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12,
102 CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13,
103 CURLINFO_FILETIME = CURLINFO_LONG + 14,
104 CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15,
105 CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16,
106 CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
107 CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18,
108 CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19,
109 CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20,
110 CURLINFO_PRIVATE = CURLINFO_STRING + 21,
111 CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22,
112 CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23,
113 CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24,
114 CURLINFO_OS_ERRNO = CURLINFO_LONG + 25,
115 CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26,
116 CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27
122 CURLMSG_NONE, /* first, not used */
123 CURLMSG_DONE, /* This easy handle has completed. 'result' contains
124 the CURLcode of the transfer */
130 CURLMSG msg; /* what this message means */
131 CURL *easy_handle; /* the handle it concerns */
134 void *whatever; /* message-specific data */
135 CURLcode result; /* return code for transfer */
141 static void (*qcurl_global_init) (long flags);
142 static void (*qcurl_global_cleanup) (void);
144 static CURL * (*qcurl_easy_init) (void);
145 static void (*qcurl_easy_cleanup) (CURL *handle);
146 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
147 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
148 static const char * (*qcurl_easy_strerror) (CURLcode);
150 static CURLM * (*qcurl_multi_init) (void);
151 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
152 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
153 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
154 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
155 static void (*qcurl_multi_cleanup) (CURLM *);
156 static const char * (*qcurl_multi_strerror) (CURLcode);
157 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
158 static void (*qcurl_slist_free_all) (curl_slist *list);
160 static dllfunction_t curlfuncs[] =
162 {"curl_global_init", (void **) &qcurl_global_init},
163 {"curl_global_cleanup", (void **) &qcurl_global_cleanup},
164 {"curl_easy_init", (void **) &qcurl_easy_init},
165 {"curl_easy_cleanup", (void **) &qcurl_easy_cleanup},
166 {"curl_easy_setopt", (void **) &qcurl_easy_setopt},
167 {"curl_easy_strerror", (void **) &qcurl_easy_strerror},
168 {"curl_easy_getinfo", (void **) &qcurl_easy_getinfo},
169 {"curl_multi_init", (void **) &qcurl_multi_init},
170 {"curl_multi_perform", (void **) &qcurl_multi_perform},
171 {"curl_multi_add_handle", (void **) &qcurl_multi_add_handle},
172 {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
173 {"curl_multi_info_read", (void **) &qcurl_multi_info_read},
174 {"curl_multi_cleanup", (void **) &qcurl_multi_cleanup},
175 {"curl_multi_strerror", (void **) &qcurl_multi_strerror},
176 {"curl_slist_append", (void **) &qcurl_slist_append},
177 {"curl_slist_free_all", (void **) &qcurl_slist_free_all},
181 // Handle for CURL DLL
182 static dllhandle_t curl_dll = NULL;
183 // will be checked at many places to find out if qcurl calls are allowed
185 typedef struct downloadinfo_s
187 char filename[MAX_OSPATH];
191 fs_offset_t startpos;
195 unsigned long bytes_received; // for buffer
196 double bytes_received_curl; // for throttling
197 double bytes_sent_curl; // for throttling
198 struct downloadinfo_s *next, *prev;
201 curl_slist *slist; // http headers
203 unsigned char *buffer;
205 curl_callback_t callback;
208 const unsigned char *postbuf;
210 const char *post_content_type;
211 const char *extraheaders;
214 static downloadinfo *downloads = NULL;
215 static int numdownloads = 0;
217 static qboolean noclear = FALSE;
219 static int numdownloads_fail = 0;
220 static int numdownloads_success = 0;
221 static int numdownloads_added = 0;
222 static char command_when_done[256] = "";
223 static char command_when_error[256] = "";
229 Sets the command which is to be executed when the last download completes AND
230 all downloads since last server connect ended with a successful status.
231 Setting the command to NULL clears it.
234 void Curl_CommandWhenDone(const char *cmd)
239 strlcpy(command_when_done, cmd, sizeof(command_when_done));
241 *command_when_done = 0;
246 Do not use yet. Not complete.
247 Problem: what counts as an error?
250 void Curl_CommandWhenError(const char *cmd)
255 strlcpy(command_when_error, cmd, sizeof(command_when_error));
257 *command_when_error = 0;
262 Curl_Clear_forthismap
264 Clears the "will disconnect on failure" flags.
267 void Curl_Clear_forthismap(void)
272 for(di = downloads; di; di = di->next)
273 di->forthismap = false;
274 Curl_CommandWhenError(NULL);
275 Curl_CommandWhenDone(NULL);
276 numdownloads_fail = 0;
277 numdownloads_success = 0;
278 numdownloads_added = 0;
285 Returns true if a download needed for the current game is running.
288 qboolean Curl_Have_forthismap(void)
290 return numdownloads_added != 0;
293 void Curl_Register_predownload(void)
295 Curl_CommandWhenDone("cl_begindownloads");
296 Curl_CommandWhenError("cl_begindownloads");
301 Curl_CheckCommandWhenDone
303 Checks if a "done command" is to be executed.
304 All downloads finished, at least one success since connect, no single failure
305 -> execute the command.
307 static void Curl_CheckCommandWhenDone(void)
311 if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
313 if(numdownloads_fail == 0)
315 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
317 Cbuf_AddText(command_when_done);
322 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
324 Cbuf_AddText(command_when_error);
327 Curl_Clear_forthismap();
338 static qboolean CURL_OpenLibrary (void)
340 const char* dllnames [] =
345 #elif defined(MACOSX)
346 "libcurl.4.dylib", // Mac OS X Notyetreleased
347 "libcurl.3.dylib", // Mac OS X Tiger
348 "libcurl.2.dylib", // Mac OS X Panther
352 "libcurl.so", // FreeBSD
362 return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
373 static void CURL_CloseLibrary (void)
375 Sys_UnloadLibrary (&curl_dll);
379 static CURLM *curlm = NULL;
380 static double bytes_received = 0; // used for bandwidth throttling
381 static double bytes_sent = 0; // used for bandwidth throttling
382 static double curltime = 0;
388 fwrite-compatible function that writes the data to a file. libcurl can call
392 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
394 fs_offset_t ret = -1;
395 size_t bytes = size * nmemb;
396 downloadinfo *di = (downloadinfo *) vdi;
400 if(di->bytes_received + bytes <= di->buffersize)
402 memcpy(di->buffer + di->bytes_received, data, bytes);
405 // otherwise: buffer overrun, ret stays -1
410 ret = FS_Write(di->stream, data, bytes);
413 di->bytes_received += bytes;
415 return ret; // why not ret / nmemb?
420 CURL_DOWNLOAD_SUCCESS = 0,
421 CURL_DOWNLOAD_FAILED,
422 CURL_DOWNLOAD_ABORTED,
423 CURL_DOWNLOAD_SERVERERROR
427 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
429 downloadinfo *di = (downloadinfo *) cbdata;
432 case CURLCBSTATUS_OK:
433 Con_DPrintf("Download of %s: OK\n", di->filename);
435 case CURLCBSTATUS_FAILED:
436 Con_DPrintf("Download of %s: FAILED\n", di->filename);
438 case CURLCBSTATUS_ABORTED:
439 Con_DPrintf("Download of %s: ABORTED\n", di->filename);
441 case CURLCBSTATUS_SERVERERROR:
442 Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
444 case CURLCBSTATUS_UNKNOWN:
445 Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
448 Con_DPrintf("Download of %s: %d\n", di->filename, status);
453 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
455 curl_default_callback(status, length_received, buffer, cbdata);
462 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
463 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
464 code from libcurl, or 0, if another error has occurred.
467 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
468 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
475 case CURL_DOWNLOAD_SUCCESS:
477 di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
479 case CURL_DOWNLOAD_FAILED:
480 di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
482 case CURL_DOWNLOAD_ABORTED:
483 di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
485 case CURL_DOWNLOAD_SERVERERROR:
486 // reopen to enforce it to have zero bytes again
489 FS_Close(di->stream);
490 di->stream = FS_OpenRealFile(di->filename, "wb", false);
494 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
498 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
504 qcurl_multi_remove_handle(curlm, di->curle);
505 qcurl_easy_cleanup(di->curle);
507 qcurl_slist_free_all(di->slist);
510 if(!di->callback && ok && !di->bytes_received)
512 Con_Printf("ERROR: empty file\n");
517 FS_Close(di->stream);
521 ok = FS_AddPack(di->filename, NULL, true);
524 // pack loading failed?
526 // better clear the file again...
527 di->stream = FS_OpenRealFile(di->filename, "wb", false);
528 FS_Close(di->stream);
530 if(di->startpos && !di->callback)
532 // this was a resume?
533 // then try to redownload it without reporting the error
534 Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->ispak, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL);
535 di->forthismap = false; // don't count the error
541 di->prev->next = di->next;
543 downloads = di->next;
545 di->next->prev = di->prev;
551 ++numdownloads_success;
562 Returns a "cleaned up" URL for display (to strip login data)
565 static const char *CleanURL(const char *url)
567 static char urlbuf[1024];
568 const char *p, *q, *r;
570 // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
571 p = strstr(url, "://");
574 q = strchr(p + 3, '@');
577 r = strchr(p + 3, '/');
580 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s%s", (int)(p - url + 3), url, q + 1);
591 CheckPendingDownloads
593 checks if there are free download slots to start new downloads in.
594 To not start too many downloads at once, only one download is added at a time,
595 up to a maximum number of cl_curl_maxdownloads are running.
598 static void CheckPendingDownloads(void)
603 if(numdownloads < cl_curl_maxdownloads.integer)
606 for(di = downloads; di; di = di->next)
612 Con_Printf("Downloading %s -> %s", CleanURL(di->url), di->filename);
614 di->stream = FS_OpenRealFile(di->filename, "ab", false);
617 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
618 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
621 FS_Seek(di->stream, 0, SEEK_END);
622 di->startpos = FS_Tell(di->stream);
625 Con_Printf(", resuming from position %ld", (long) di->startpos);
630 Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url));
634 di->curle = qcurl_easy_init();
636 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
637 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
638 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
639 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
640 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
641 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
642 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
643 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
644 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
645 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
646 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
647 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
649 Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
650 //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
652 if(di->post_content_type)
654 qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
655 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
656 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
657 di->slist = qcurl_slist_append(di->slist, va("Content-Type: %s", di->post_content_type));
660 // parse extra headers into slist
661 // \n separated list!
662 h = di->extraheaders;
665 const char *hh = strchr(h, '\n');
668 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
669 memcpy(buf, h, hh - h);
671 di->slist = qcurl_slist_append(di->slist, buf);
676 di->slist = qcurl_slist_append(di->slist, h);
681 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
683 qcurl_multi_add_handle(curlm, di->curle);
686 if(numdownloads >= cl_curl_maxdownloads.integer)
697 this function MUST be called before using anything else in this file.
698 On Win32, this must be called AFTER WSAStartup has been done!
706 qcurl_global_init(CURL_GLOBAL_NOTHING);
707 curlm = qcurl_multi_init();
714 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
717 void Curl_ClearRequirements(void);
718 void Curl_Shutdown(void)
722 Curl_ClearRequirements();
732 Finds the internal information block for a download given by file name.
735 static downloadinfo *Curl_Find(const char *filename)
740 for(di = downloads; di; di = di->next)
741 if(!strcasecmp(di->filename, filename))
746 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
751 for(di = downloads; di; )
753 if(di->callback == callback && di->callback_data == cbdata)
755 di->callback = curl_quiet_callback; // do NOT call the callback
756 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
768 Starts a download of a given URL to the file name portion of this URL (or name
769 if given) in the "dlcache/" folder.
772 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
786 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
787 p = strchr(URL, ':');
790 if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
792 char addressstring[128];
794 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
795 q = strchr(addressstring, ':');
797 q = addressstring + strlen(addressstring);
800 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
806 // Note: This extraction of the file name portion is NOT entirely correct.
808 // It does the following:
810 // http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
811 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
812 // http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
814 // However, I'd like to keep this "buggy" behavior so that PHP script
815 // authors can write download scripts without having to enable
816 // AcceptPathInfo on Apache. They just have to ensure that their script
817 // can be called with such a "fake" path name like
818 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
820 // By the way, such PHP scripts should either send the file or a
821 // "Location:" redirect; PHP code example:
823 // header("Location: http://www.example.com/");
825 // By the way, this will set User-Agent to something like
826 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
827 // dp://serverhost:serverport/ so you can filter on this; an example
828 // httpd log file line might be:
830 // 141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
833 name = CleanURL(URL);
837 p = strrchr(name, '/');
838 p = p ? (p+1) : name;
840 length = q ? (size_t)(q - p) : strlen(p);
841 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
843 name = fn; // make it point back
845 // already downloading the file?
847 downloadinfo *di = Curl_Find(fn);
850 Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url));
852 // however, if it was not for this map yet...
853 if(forthismap && !di->forthismap)
855 di->forthismap = true;
856 // this "fakes" a download attempt so the client will wait for
857 // the download to finish and then reconnect
858 ++numdownloads_added;
865 if(ispak && FS_FileExists(fn))
867 qboolean already_loaded;
868 if(FS_AddPack(fn, &already_loaded, true))
870 Con_DPrintf("%s already exists, not downloading!\n", fn);
872 Con_DPrintf("(pak was already loaded)\n");
877 ++numdownloads_added;
878 ++numdownloads_success;
886 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
890 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
892 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
894 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
896 f = FS_OpenRealFile(fn, "wb", false);
910 // if we get here, we actually want to download... so first verify the
911 // URL scheme (so one can't read local files using file://)
912 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
914 Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
919 ++numdownloads_added;
920 di = (downloadinfo *) Z_Malloc(sizeof(*di));
921 strlcpy(di->filename, name, sizeof(di->filename));
922 strlcpy(di->url, URL, sizeof(di->url));
923 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
924 di->forthismap = forthismap;
929 di->ispak = (ispak && !buf);
930 di->maxspeed = maxspeed;
931 di->bytes_received = 0;
932 di->bytes_received_curl = 0;
933 di->bytes_sent_curl = 0;
934 di->extraheaders = extraheaders;
935 di->next = downloads;
941 di->buffersize = bufsize;
944 di->callback = curl_default_callback;
945 di->callback_data = di;
949 di->callback = callback;
950 di->callback_data = cbdata;
953 if(post_content_type)
955 di->post_content_type = post_content_type;
956 di->postbuf = postbuf;
957 di->postbufsize = postbufsize;
961 di->post_content_type = NULL;
971 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap)
973 return Curl_Begin(URL, NULL, maxspeed, name, ispak, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
975 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
977 return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
979 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
981 return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
988 call this regularily as this will always download as much as possible without
999 if(!cl_curl_enabled.integer)
1005 Curl_CheckCommandWhenDone();
1010 if(realtime < curltime) // throttle
1019 mc = qcurl_multi_perform(curlm, &remaining);
1021 while(mc == CURLM_CALL_MULTI_PERFORM);
1023 for(di = downloads; di; di = di->next)
1028 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1029 bytes_sent += (b - di->bytes_sent_curl);
1030 di->bytes_sent_curl = b;
1031 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1032 bytes_sent += (b - di->bytes_received_curl);
1033 di->bytes_received_curl = b;
1039 CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1042 if(msg->msg == CURLMSG_DONE)
1044 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1046 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1047 result = msg->data.result;
1050 failed = CURL_DOWNLOAD_FAILED;
1055 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1058 case 4: // e.g. 404?
1059 case 5: // e.g. 500?
1060 failed = CURL_DOWNLOAD_SERVERERROR;
1061 result = (CURLcode) code;
1066 Curl_EndDownload(di, failed, result);
1071 CheckPendingDownloads();
1073 // when will we curl the next time?
1074 // we will wait a bit to ensure our download rate is kept.
1075 // we now know that realtime >= curltime... so set up a new curltime
1077 // use the slowest allowing download to derive the maxspeed... this CAN
1078 // be done better, but maybe later
1079 maxspeed = cl_curl_maxspeed.value;
1080 for(di = downloads; di; di = di->next)
1081 if(di->maxspeed > 0)
1082 if(di->maxspeed < maxspeed || maxspeed <= 0)
1083 maxspeed = di->maxspeed;
1087 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1088 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1093 curltime = realtime;
1097 ====================
1100 Stops ALL downloads.
1101 ====================
1103 void Curl_CancelAll(void)
1110 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1111 // INVARIANT: downloads will point to the next download after that!
1116 ====================
1119 returns true iff there is a download running.
1120 ====================
1122 qboolean Curl_Running(void)
1127 return downloads != NULL;
1131 ====================
1132 Curl_GetDownloadAmount
1134 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1135 for the given download.
1136 ====================
1138 static double Curl_GetDownloadAmount(downloadinfo *di)
1145 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1147 return (di->startpos + di->bytes_received) / (di->startpos + length);
1156 ====================
1157 Curl_GetDownloadSpeed
1159 returns the speed of the given download in bytes per second
1160 ====================
1162 static double Curl_GetDownloadSpeed(downloadinfo *di)
1169 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1177 ====================
1180 prints the download list
1181 ====================
1183 // TODO rewrite using Curl_GetDownloadInfo?
1184 static void Curl_Info_f(void)
1191 Con_Print("Currently running downloads:\n");
1192 for(di = downloads; di; di = di->next)
1194 double speed, percent;
1195 Con_Printf(" %s -> %s ", CleanURL(di->url), di->filename);
1196 percent = 100.0 * Curl_GetDownloadAmount(di);
1197 speed = Curl_GetDownloadSpeed(di);
1199 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1201 Con_Print("(queued)\n");
1206 Con_Print("No downloads running.\n");
1211 ====================
1214 implements the "curl" console command
1218 curl --cancel filename
1223 curl [--pak] [--forthismap] [--for filename filename...] url
1224 --pak: after downloading, load the package into the virtual file system
1225 --for filename...: only download of at least one of the named files is missing
1226 --forthismap: don't reconnect on failure
1228 curl --clear_autodownload
1229 clears the download success/failure counters
1231 curl --finish_autodownload
1232 if at least one download has been started, disconnect and drop to the menu
1233 once the last download completes successfully, reconnect to the current server
1234 ====================
1236 void Curl_Curl_f(void)
1238 double maxspeed = 0;
1241 qboolean pak = false;
1242 qboolean forthismap = false;
1244 const char *name = 0;
1248 Con_Print("libcurl DLL not found, this command is inactive.\n");
1252 if(!cl_curl_enabled.integer)
1254 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1260 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1264 url = Cmd_Argv(Cmd_Argc() - 1);
1267 for(i = 1; i != end; ++i)
1269 const char *a = Cmd_Argv(i);
1270 if(!strcmp(a, "--info"))
1275 else if(!strcmp(a, "--cancel"))
1277 if(i == end - 1) // last argument
1281 downloadinfo *di = Curl_Find(url);
1283 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1285 Con_Print("download not found\n");
1289 else if(!strcmp(a, "--pak"))
1293 else if(!strcmp(a, "--for")) // must be last option
1295 for(i = i + 1; i != end - 1; ++i)
1297 if(!FS_FileExists(Cmd_Argv(i)))
1298 goto needthefile; // why can't I have a "double break"?
1300 // if we get here, we have all the files...
1303 else if(!strcmp(a, "--forthismap"))
1307 else if(!strcmp(a, "--as"))
1315 else if(!strcmp(a, "--clear_autodownload"))
1317 // mark all running downloads as "not for this map", so if they
1318 // fail, it does not matter
1319 Curl_Clear_forthismap();
1322 else if(!strcmp(a, "--finish_autodownload"))
1324 if(numdownloads_added)
1326 char donecommand[256];
1329 if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1331 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1332 Curl_CommandWhenDone(donecommand);
1336 Curl_CheckCommandWhenDone();
1339 Curl_Register_predownload();
1344 else if(!strncmp(a, "--maxspeed=", 11))
1346 maxspeed = atof(a + 11);
1350 Con_Printf("curl: invalid option %s\n", a);
1351 // but we ignore the option
1356 Curl_Begin_ToFile(url, maxspeed, name, pak, forthismap);
1360 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1362 Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1366 void Curl_CurlCat_f(void)
1369 const char *url = Cmd_Argv(1);
1370 buf = Z_Malloc(16384);
1371 Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1376 ====================
1379 loads the commands and cvars this library uses
1380 ====================
1382 void Curl_Init_Commands(void)
1384 Cvar_RegisterVariable (&cl_curl_enabled);
1385 Cvar_RegisterVariable (&cl_curl_maxdownloads);
1386 Cvar_RegisterVariable (&cl_curl_maxspeed);
1387 Cvar_RegisterVariable (&sv_curl_defaulturl);
1388 Cvar_RegisterVariable (&sv_curl_serverpackages);
1389 Cvar_RegisterVariable (&sv_curl_maxspeed);
1390 Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1391 //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1395 ====================
1396 Curl_GetDownloadInfo
1398 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1399 The number of elements in the array is returned in int *nDownloads.
1400 const char **additional_info may be set to a string of additional user
1401 information, or to NULL if no such display shall occur. The returned
1402 array must be freed later using Z_Free.
1403 ====================
1405 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1409 Curl_downloadinfo_t *downinfo;
1410 static char addinfo[128];
1416 *additional_info = NULL;
1421 for(di = downloads; di; di = di->next)
1424 downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1426 for(di = downloads; di; di = di->next)
1428 // do not show infobars for background downloads
1429 if(developer.integer <= 0)
1432 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1435 downinfo[i].progress = Curl_GetDownloadAmount(di);
1436 downinfo[i].speed = Curl_GetDownloadSpeed(di);
1437 downinfo[i].queued = false;
1441 downinfo[i].queued = true;
1448 // TODO: can I clear command_when_done as soon as the first download fails?
1449 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1451 if(!strncmp(command_when_done, "connect ", 8))
1452 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1453 else if(!strcmp(command_when_done, "cl_begindownloads"))
1454 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1456 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1457 *additional_info = addinfo;
1460 *additional_info = NULL;
1469 ====================
1472 finds the URL where to find a given package.
1474 For this, it reads a file "curl_urls.txt" of the following format:
1477 revdm*.pk3 http://revdm/downloads/are/here/
1478 * http://any/other/stuff/is/here/
1480 The URLs should end in /. If not, downloads will still work, but the cached files
1481 can't be just put into the data directory with the same download configuration
1482 (you might want to do this if you want to tag downloaded files from your
1483 server, but you should not). "-" means "don't download".
1485 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1488 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1489 this file for obvious reasons.
1490 ====================
1492 static const char *Curl_FindPackURL(const char *filename)
1494 static char foundurl[1024];
1495 fs_offset_t filesize;
1496 char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1499 // read lines of format "pattern url"
1501 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1502 qboolean eof = false;
1514 if(pattern && url && patternend)
1520 if(matchpattern(filename, pattern, true))
1522 strlcpy(foundurl, url, sizeof(foundurl));
1534 if(pattern && !patternend)
1536 else if(url && !urlend)
1542 else if(pattern && patternend && !url)
1551 return sv_curl_defaulturl.string;
1554 typedef struct requirement_s
1556 struct requirement_s *next;
1557 char filename[MAX_OSPATH];
1560 static requirement *requirements = NULL;
1564 ====================
1567 Adds the given file to the list of requirements.
1568 ====================
1570 void Curl_RequireFile(const char *filename)
1572 requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1573 req->next = requirements;
1574 strlcpy(req->filename, filename, sizeof(req->filename));
1579 ====================
1580 Curl_ClearRequirements
1582 Clears the list of required files for playing on the current map.
1583 This should be called at every map change.
1584 ====================
1586 void Curl_ClearRequirements(void)
1590 requirement *req = requirements;
1591 requirements = requirements->next;
1597 ====================
1598 Curl_SendRequirements
1600 Makes the current host_clients download all files he needs.
1601 This is done by sending him the following console commands:
1603 curl --clear_autodownload
1604 curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1605 curl --finish_autodownload
1606 ====================
1608 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1611 const char *thispack = FS_WhichPack(filename);
1612 const char *packurl;
1617 p = strrchr(thispack, '/');
1621 packurl = Curl_FindPackURL(thispack);
1623 if(packurl && *packurl && strcmp(packurl, "-"))
1626 strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1628 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1629 strlcat(sendbuffer, thispack, sendbuffer_len);
1630 if(sv_curl_maxspeed.value > 0)
1631 dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1632 strlcat(sendbuffer, " --for ", sendbuffer_len);
1633 strlcat(sendbuffer, filename, sendbuffer_len);
1634 strlcat(sendbuffer, " ", sendbuffer_len);
1635 strlcat(sendbuffer, packurl, sendbuffer_len);
1636 strlcat(sendbuffer, thispack, sendbuffer_len);
1637 strlcat(sendbuffer, "\n", sendbuffer_len);
1644 void Curl_SendRequirements(void)
1646 // for each requirement, find the pack name
1647 char sendbuffer[4096] = "";
1649 qboolean foundone = false;
1652 for(req = requirements; req; req = req->next)
1653 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1655 p = sv_curl_serverpackages.string;
1656 while(COM_ParseToken_Simple(&p, false, false))
1657 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1660 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1662 if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1663 Host_ClientCommands("%s", sendbuffer);
1665 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");