X-Git-Url: http://de.git.xonotic.org/?a=blobdiff_plain;ds=sidebyside;f=libcurl.c;h=085dd415d78d841856c3d258188be4d5fa142ff7;hb=417727143082992c3b4c9469f0c5683005c949fd;hp=28240a82eeb923a33b07e4d15ab6975fb541584e;hpb=2cbfd57208b7e0050bf532456dedae601670af88;p=xonotic%2Fdarkplaces.git diff --git a/libcurl.c b/libcurl.c index 28240a82..085dd415 100644 --- a/libcurl.c +++ b/libcurl.c @@ -128,15 +128,15 @@ typedef struct CURLMsg; static void (*qcurl_global_init) (long flags); -static void (*qcurl_global_cleanup) (); +static void (*qcurl_global_cleanup) (void); -static CURL * (*qcurl_easy_init) (); +static CURL * (*qcurl_easy_init) (void); static void (*qcurl_easy_cleanup) (CURL *handle); static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...); static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...); static const char * (*qcurl_easy_strerror) (CURLcode); -static CURLM * (*qcurl_multi_init) (); +static CURLM * (*qcurl_multi_init) (void); static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles); static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle); static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle); @@ -169,8 +169,8 @@ static dllhandle_t curl_dll = NULL; typedef struct downloadinfo_s { - char filename[MAX_QPATH]; - char url[256]; + char filename[MAX_OSPATH]; + char url[1024]; char referer[256]; qfile_t *stream; fs_offset_t startpos; @@ -180,6 +180,11 @@ typedef struct downloadinfo_s unsigned long bytes_received; struct downloadinfo_s *next, *prev; qboolean forthismap; + + unsigned char *buffer; + size_t buffersize; + curl_callback_t callback; + void *callback_data; } downloadinfo; static downloadinfo *downloads = NULL; @@ -235,7 +240,7 @@ Curl_Clear_forthismap Clears the "will disconnect on failure" flags. ==================== */ -void Curl_Clear_forthismap() +void Curl_Clear_forthismap(void) { downloadinfo *di; if(noclear) @@ -256,12 +261,12 @@ Curl_Have_forthismap Returns true if a download needed for the current game is running. ==================== */ -qboolean Curl_Have_forthismap() +qboolean Curl_Have_forthismap(void) { - return numdownloads_added; + return numdownloads_added != 0; } -void Curl_Register_predownload() +void Curl_Register_predownload(void) { Curl_CommandWhenDone("cl_begindownloads"); Curl_CommandWhenError("cl_begindownloads"); @@ -275,7 +280,7 @@ Checks if a "done command" is to be executed. All downloads finished, at least one success since connect, no single failure -> execute the command. */ -static void Curl_CheckCommandWhenDone() +static void Curl_CheckCommandWhenDone(void) { if(!curl_dll) return; @@ -361,15 +366,28 @@ this. */ static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi) { - fs_offset_t ret; + fs_offset_t ret = -1; size_t bytes = size * nmemb; downloadinfo *di = (downloadinfo *) vdi; + if(di->buffer) + { + if(di->bytes_received + bytes <= di->buffersize) + { + memcpy(di->buffer + di->bytes_received, data, bytes); + ret = bytes; + } + // otherwise: buffer overrun, ret stays -1 + } + + if(di->stream) + { + ret = FS_Write(di->stream, data, bytes); + } + bytes_received += bytes; di->bytes_received += bytes; - ret = FS_Write(di->stream, data, bytes); - return ret; // why not ret / nmemb? } @@ -382,6 +400,38 @@ typedef enum } CurlStatus; +static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata) +{ + downloadinfo *di = (downloadinfo *) cbdata; + switch(status) + { + case CURLCBSTATUS_OK: + Con_Printf("Download of %s: OK\n", di->filename); + break; + case CURLCBSTATUS_FAILED: + Con_Printf("Download of %s: FAILED\n", di->filename); + break; + case CURLCBSTATUS_ABORTED: + Con_Printf("Download of %s: ABORTED\n", di->filename); + break; + case CURLCBSTATUS_SERVERERROR: + Con_Printf("Download of %s: (unknown server error)\n", di->filename); + break; + case CURLCBSTATUS_UNKNOWN: + Con_Printf("Download of %s: (unknown client error)\n", di->filename); + break; + default: + Con_Printf("Download of %s: %d\n", di->filename, status); + break; + } +} + +static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata) +{ + if(developer.integer) + curl_default_callback(status, length_received, buffer, cbdata); +} + /* ==================== Curl_EndDownload @@ -391,6 +441,7 @@ CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error code from libcurl, or 0, if another error has occurred. ==================== */ +static qboolean Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata); static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error) { qboolean ok = false; @@ -399,24 +450,29 @@ static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error switch(status) { case CURL_DOWNLOAD_SUCCESS: - Con_Printf("Download of %s: OK\n", di->filename); ok = true; + di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data); break; case CURL_DOWNLOAD_FAILED: - Con_Printf("Download of %s: FAILED\n", di->filename); - if(error) - Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error)); + di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data); break; case CURL_DOWNLOAD_ABORTED: - Con_Printf("Download of %s: ABORTED\n", di->filename); + di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data); break; case CURL_DOWNLOAD_SERVERERROR: - Con_Printf("Download of %s: %d\n", di->filename, (int) error); - // reopen to enforce it to have zero bytes again - FS_Close(di->stream); - di->stream = FS_OpenRealFile(di->filename, "wb", false); + if(di->stream) + { + FS_Close(di->stream); + di->stream = FS_OpenRealFile(di->filename, "wb", false); + } + if(di->callback) + di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data); + break; + default: + if(di->callback) + di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data); break; } @@ -426,7 +482,7 @@ static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error qcurl_easy_cleanup(di->curle); } - if(ok && !di->bytes_received) + if(!di->callback && ok && !di->bytes_received) { Con_Printf("ERROR: empty file\n"); ok = false; @@ -436,7 +492,25 @@ static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error FS_Close(di->stream); if(ok && di->ispak) + { ok = FS_AddPack(di->filename, NULL, true); + if(!ok) + { + // pack loading failed? + // this is critical + // better clear the file again... + di->stream = FS_OpenRealFile(di->filename, "wb", false); + FS_Close(di->stream); + + if(di->startpos && !di->callback) + { + // this was a resume? + // then try to redownload it without reporting the error + Curl_Begin(di->url, di->filename, di->ispak, di->forthismap, NULL, 0, NULL, NULL); + di->forthismap = false; // don't count the error + } + } + } if(di->prev) di->prev->next = di->next; @@ -465,7 +539,7 @@ To not start too many downloads at once, only one download is added at a time, up to a maximum number of cl_curl_maxdownloads are running. ==================== */ -static void CheckPendingDownloads() +static void CheckPendingDownloads(void) { if(!curl_dll) return; @@ -476,22 +550,30 @@ static void CheckPendingDownloads() { if(!di->started) { - Con_Printf("Downloading %s -> %s", di->url, di->filename); + if(!di->buffer) + { + Con_Printf("Downloading %s -> %s", di->url, di->filename); - di->stream = FS_OpenRealFile(di->filename, "ab", false); - if(!di->stream) + di->stream = FS_OpenRealFile(di->filename, "ab", false); + if(!di->stream) + { + Con_Printf("\nFAILED: Could not open output file %s\n", di->filename); + Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK); + return; + } + FS_Seek(di->stream, 0, SEEK_END); + di->startpos = FS_Tell(di->stream); + + if(di->startpos > 0) + Con_Printf(", resuming from position %ld", (long) di->startpos); + Con_Print("...\n"); + } + else { - Con_Printf("\nFAILED: Could not open output file %s\n", di->filename); - Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK); - return; + Con_DPrintf("Downloading %s -> memory\n", di->url); + di->startpos = 0; } - FS_Seek(di->stream, 0, SEEK_END); - di->startpos = FS_Tell(di->stream); - if(di->startpos > 0) - Con_Printf(", resuming from position %ld", (long) di->startpos); - Con_Print("...\n"); - di->curle = qcurl_easy_init(); qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url); qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion); @@ -521,7 +603,7 @@ this function MUST be called before using anything else in this file. On Win32, this must be called AFTER WSAStartup has been done! ==================== */ -void Curl_Init() +void Curl_Init(void) { CURL_OpenLibrary(); if(!curl_dll) @@ -537,8 +619,8 @@ Curl_Shutdown Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET. ==================== */ -void Curl_ClearRequirements(); -void Curl_Shutdown() +void Curl_ClearRequirements(void); +void Curl_Shutdown(void) { if(!curl_dll) return; @@ -566,6 +648,24 @@ static downloadinfo *Curl_Find(const char *filename) return NULL; } +void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata) +{ + downloadinfo *di; + if(!curl_dll) + return; + for(di = downloads; di; ) + { + if(di->callback == callback && di->callback_data == cbdata) + { + di->callback = curl_quiet_callback; // do NOT call the callback + Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK); + di = downloads; + } + else + di = di->next; + } +} + /* ==================== Curl_Begin @@ -574,13 +674,15 @@ Starts a download of a given URL to the file name portion of this URL (or name if given) in the "dlcache/" folder. ==================== */ -void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap) +static qboolean Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata) { if(!curl_dll) - return; + { + return false; + } else { - char fn[MAX_QPATH]; + char fn[MAX_OSPATH]; const char *p, *q; size_t length; downloadinfo *di; @@ -613,79 +715,94 @@ void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean fort if(!name) name = URL; - p = strrchr(name, '/'); - p = p ? (p+1) : name; - q = strchr(p, '?'); - length = q ? (size_t)(q - p) : strlen(p); - dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p); - // already downloading the file? + if(!buf) { - downloadinfo *di = Curl_Find(fn); - if(di) - { - Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url); - - // however, if it was not for this map yet... - if(forthismap && !di->forthismap) - { - di->forthismap = true; - // this "fakes" a download attempt so the client will wait for - // the download to finish and then reconnect - ++numdownloads_added; - } + p = strrchr(name, '/'); + p = p ? (p+1) : name; + q = strchr(p, '?'); + length = q ? (size_t)(q - p) : strlen(p); + dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p); - return; - } - } + name = fn; // make it point back - if(ispak && FS_FileExists(fn)) - { - qboolean already_loaded; - if(FS_AddPack(fn, &already_loaded, true)) + // already downloading the file? { - Con_DPrintf("%s already exists, not downloading!\n", fn); - if(already_loaded) - Con_DPrintf("(pak was already loaded)\n"); - else + downloadinfo *di = Curl_Find(fn); + if(di) { - if(forthismap) + Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url); + + // however, if it was not for this map yet... + if(forthismap && !di->forthismap) { + di->forthismap = true; + // this "fakes" a download attempt so the client will wait for + // the download to finish and then reconnect ++numdownloads_added; - ++numdownloads_success; } + + return false; } - return; } - else + + if(ispak && FS_FileExists(fn)) { - qfile_t *f = FS_OpenVirtualFile(fn, false); - if(f) + qboolean already_loaded; + if(FS_AddPack(fn, &already_loaded, true)) { - char buf[4] = {0}; - FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp - - if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4)) + Con_DPrintf("%s already exists, not downloading!\n", fn); + if(already_loaded) + Con_DPrintf("(pak was already loaded)\n"); + else { - Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn); - FS_Close(f); - f = FS_OpenRealFile(fn, "wb", false); - if(f) - FS_Close(f); + if(forthismap) + { + ++numdownloads_added; + ++numdownloads_success; + } } - else + + return false; + } + else + { + qfile_t *f = FS_OpenRealFile(fn, "rb", false); + if(f) { - // OK - FS_Close(f); + char buf[4] = {0}; + FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp + + if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4)) + { + Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn); + FS_Close(f); + f = FS_OpenRealFile(fn, "wb", false); + if(f) + FS_Close(f); + } + else + { + // OK + FS_Close(f); + } } } } } + // if we get here, we actually want to download... so first verify the + // URL scheme (so one can't read local files using file://) + if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8)) + { + Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL); + return false; + } + if(forthismap) ++numdownloads_added; di = (downloadinfo *) Z_Malloc(sizeof(*di)); - strlcpy(di->filename, fn, sizeof(di->filename)); + strlcpy(di->filename, name, sizeof(di->filename)); strlcpy(di->url, URL, sizeof(di->url)); dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid"); di->forthismap = forthismap; @@ -693,16 +810,39 @@ void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean fort di->startpos = 0; di->curle = NULL; di->started = false; - di->ispak = ispak; + di->ispak = (ispak && !buf); di->bytes_received = 0; di->next = downloads; di->prev = NULL; if(di->next) di->next->prev = di; + + di->buffer = buf; + di->buffersize = bufsize; + if(callback == NULL) + { + di->callback = curl_default_callback; + di->callback_data = di; + } + else + { + di->callback = callback; + di->callback_data = cbdata; + } + downloads = di; + return true; } } +qboolean Curl_Begin_ToFile(const char *URL, const char *name, qboolean ispak, qboolean forthismap) +{ + return Curl_Begin(URL, name, ispak, forthismap, NULL, 0, NULL, NULL); +} +qboolean Curl_Begin_ToMemory(const char *URL, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata) +{ + return Curl_Begin(URL, NULL, false, false, buf, bufsize, callback, cbdata); +} /* ==================== @@ -712,7 +852,7 @@ call this regularily as this will always download as much as possible without blocking. ==================== */ -void Curl_Run() +void Curl_Run(void) { noclear = FALSE; @@ -765,7 +905,7 @@ void Curl_Run() case 4: // e.g. 404? case 5: // e.g. 500? failed = CURL_DOWNLOAD_SERVERERROR; - result = code; + result = (CURLcode) code; break; } } @@ -797,7 +937,7 @@ Curl_CancelAll Stops ALL downloads. ==================== */ -void Curl_CancelAll() +void Curl_CancelAll(void) { if(!curl_dll) return; @@ -816,7 +956,7 @@ Curl_Running returns true iff there is a download running. ==================== */ -qboolean Curl_Running() +qboolean Curl_Running(void) { if(!curl_dll) return false; @@ -841,7 +981,7 @@ static double Curl_GetDownloadAmount(downloadinfo *di) double length; qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length); if(length > 0) - return di->bytes_received / length; + return (di->startpos + di->bytes_received) / (di->startpos + length); else return 0; } @@ -878,7 +1018,7 @@ prints the download list ==================== */ // TODO rewrite using Curl_GetDownloadInfo? -static void Curl_Info_f() +static void Curl_Info_f(void) { downloadinfo *di; if(!curl_dll) @@ -1049,9 +1189,25 @@ void Curl_Curl_f(void) } needthefile: - Curl_Begin(url, name, pak, forthismap); + Curl_Begin_ToFile(url, name, pak, forthismap); } +/* +static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata) +{ + Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer); + Z_Free(buffer); +} + +void Curl_CurlCat_f(void) +{ + unsigned char *buf; + const char *url = Cmd_Argv(1); + buf = Z_Malloc(16384); + Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL); +} +*/ + /* ==================== Curl_Init_Commands @@ -1067,6 +1223,7 @@ void Curl_Init_Commands(void) Cvar_RegisterVariable (&sv_curl_defaulturl); Cvar_RegisterVariable (&sv_curl_serverpackages); Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path"); + //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)"); } /* @@ -1082,7 +1239,7 @@ array must be freed later using Z_Free. */ Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info) { - int n, i; + int i; downloadinfo *di; Curl_downloadinfo_t *downinfo; static char addinfo[128]; @@ -1095,14 +1252,18 @@ Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **addition return NULL; } - n = 0; + i = 0; for(di = downloads; di; di = di->next) - ++n; + ++i; - downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n); + downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i); i = 0; for(di = downloads; di; di = di->next) { + // do not show infobars for background downloads + if(!developer.integer) + if(di->buffer) + continue; strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename)); if(di->curle) { @@ -1134,7 +1295,7 @@ Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **addition *additional_info = NULL; } - *nDownloads = n; + *nDownloads = i; return downinfo; } @@ -1165,7 +1326,7 @@ this file for obvious reasons. */ static const char *Curl_FindPackURL(const char *filename) { - static char foundurl[256]; + static char foundurl[1024]; fs_offset_t filesize; char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize); if(buf && filesize) @@ -1228,7 +1389,7 @@ static const char *Curl_FindPackURL(const char *filename) typedef struct requirement_s { struct requirement_s *next; - char filename[MAX_QPATH]; + char filename[MAX_OSPATH]; } requirement; static requirement *requirements = NULL; @@ -1257,7 +1418,7 @@ Clears the list of required files for playing on the current map. This should be called at every map change. ==================== */ -void Curl_ClearRequirements() +void Curl_ClearRequirements(void) { const char *p; while(requirements) @@ -1287,7 +1448,7 @@ This is done by sending him the following console commands: curl --finish_autodownload ==================== */ -void Curl_SendRequirements() +void Curl_SendRequirements(void) { // for each requirement, find the pack name char sendbuffer[4096] = "";