qcsrc/lib/urllib.qc

   1 #include "urllib.qh"
   2
   3 // files
   4 .float url_fh;
   5 const float URL_FH_CURL = -1;
   6 const float URL_FH_STDOUT = -2;
   7
   8 // URLs
   9 .string url_url;
  10 .float url_wbuf;
  11 .float url_wbufpos;
  12 .float url_rbuf;
  13 .float url_rbufpos;
  14 .float url_id;
  15 .url_ready_func url_ready;
  16 .entity url_ready_pass;
  17
  18 // for multi handles
  19 .int url_attempt;
  20 .int url_mode;
  21
  22 entity url_fromid[NUM_URL_ID];
  23 int autocvar__urllib_nextslot;
  24
  25 float url_URI_Get_Callback(int id, float status, string data)
  26 {
  27         if (id < MIN_URL_ID) return 0;
  28         id -= MIN_URL_ID;
  29         if (id >= NUM_URL_ID) return 0;
  30         entity e;
  31         e = url_fromid[id];
  32         if (!e) return 0;
  33         if (e.url_rbuf >= 0 || e.url_wbuf >= 0)
  34         {
  35                 LOG_INFOF("WARNING: handle %d (%s) has already received data?!?\n", id + NUM_URL_ID, e.url_url);
  36                 return 0;
  37         }
  38
  39         // whatever happens, we will remove the URL from the list of IDs
  40         url_fromid[id] = NULL;
  41
  42         // if we get here, we MUST have both buffers cleared
  43         if (e.url_rbuf != -1 || e.url_wbuf != -1 || e.url_fh != URL_FH_CURL) error("url_URI_Get_Callback: not a request waiting for data");
  44
  45         if (status == 0)
  46         {
  47                 // WE GOT DATA!
  48                 float n, i;
  49                 n = tokenizebyseparator(data, "\n");
  50                 e.url_rbuf = buf_create();
  51                 if (e.url_rbuf < 0)
  52                 {
  53                         LOG_INFO("url_URI_Get_Callback: out of memory in buf_create\n");
  54                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  55                         strunzone(e.url_url);
  56                         remove(e);
  57                         return 1;
  58                 }
  59                 e.url_rbufpos = 0;
  60                 if (e.url_rbuf < 0)
  61                 {
  62                         LOG_INFO("url_URI_Get_Callback: out of memory in buf_create\n");
  63                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  64                         strunzone(e.url_url);
  65                         remove(e);
  66                         return 1;
  67                 }
  68                 for (i = 0; i < n; ++i)
  69                         bufstr_set(e.url_rbuf, i, argv(i));
  70                 e.url_ready(e, e.url_ready_pass, URL_READY_CANREAD);
  71                 return 1;
  72         }
  73         else
  74         {
  75                 // an ERROR
  76                 e.url_ready(e, e.url_ready_pass, -fabs(status));
  77                 strunzone(e.url_url);
  78                 remove(e);
  79                 return 1;
  80         }
  81 }
  82
  83 void url_single_fopen(string url, int mode, url_ready_func rdy, entity pass)
  84 {
  85         entity e;
  86         int i;
  87         if (strstrofs(url, "://", 0) >= 0)
  88         {
  89                 switch (mode)
  90                 {
  91                         case FILE_WRITE:
  92                         case FILE_APPEND:
  93                                 // collect data to a stringbuffer for a POST request
  94                                 // attempts to close will result in a reading handle
  95
  96                                 // create a writing end that does nothing yet
  97                                 e = spawn();
  98                                 e.classname = "url_single_fopen_file";
  99                                 e.url_url = strzone(url);
 100                                 e.url_fh = URL_FH_CURL;
 101                                 e.url_wbuf = buf_create();
 102                                 if (e.url_wbuf < 0)
 103                                 {
 104                                         LOG_INFO("url_single_fopen: out of memory in buf_create\n");
 105                                         rdy(e, pass, URL_READY_ERROR);
 106                                         strunzone(e.url_url);
 107                                         remove(e);
 108                                         return;
 109                                 }
 110                                 e.url_wbufpos = 0;
 111                                 e.url_rbuf = -1;
 112                                 e.url_ready = rdy;
 113                                 e.url_ready_pass = pass;
 114                                 rdy(e, pass, URL_READY_CANWRITE);
 115                                 break;
 116
 117                         case FILE_READ:
 118                                 // read data only
 119
 120                                 // get slot for HTTP request
 121                                 for (i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 122                                         if (url_fromid[i] == NULL) break;
 123                                 if (i >= NUM_URL_ID)
 124                                 {
 125                                         for (i = 0; i < autocvar__urllib_nextslot; ++i)
 126                                                 if (url_fromid[i] == NULL) break;
 127                                         if (i >= autocvar__urllib_nextslot)
 128                                         {
 129                                                 LOG_INFO("url_single_fopen: too many concurrent requests\n");
 130                                                 rdy(NULL, pass, URL_READY_ERROR);
 131                                                 return;
 132                                         }
 133                                 }
 134
 135                                 // GET the data
 136                                 if (!crypto_uri_postbuf(url, i + MIN_URL_ID, string_null, string_null, -1, 0))
 137                                 {
 138                                         LOG_INFO("url_single_fopen: failure in crypto_uri_postbuf\n");
 139                                         rdy(NULL, pass, URL_READY_ERROR);
 140                                         return;
 141                                 }
 142
 143                                 // Make a dummy handle object (no buffers at
 144                                 // all). Wait for data to come from the
 145                                 // server, then call the callback
 146                                 e = spawn();
 147                                 e.classname = "url_single_fopen_file";
 148                                 e.url_url = strzone(url);
 149                                 e.url_fh = URL_FH_CURL;
 150                                 e.url_rbuf = -1;
 151                                 e.url_wbuf = -1;
 152                                 e.url_ready = rdy;
 153                                 e.url_ready_pass = pass;
 154                                 e.url_id = i;
 155                                 url_fromid[i] = e;
 156
 157                                 // make sure this slot won't be reused quickly even on map change
 158                                 cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 159                                 break;
 160                 }
 161         }
 162         else if (url == "-")
 163         {
 164                 switch (mode)
 165                 {
 166                         case FILE_WRITE:
 167                         case FILE_APPEND:
 168                                 e = spawn();
 169                                 e.classname = "url_single_fopen_stdout";
 170                                 e.url_fh = URL_FH_STDOUT;
 171                                 e.url_ready = rdy;
 172                                 e.url_ready_pass = pass;
 173                                 rdy(e, pass, URL_READY_CANWRITE);
 174                                 break;
 175                         case FILE_READ:
 176                                 LOG_INFO("url_single_fopen: cannot open '-' for reading\n");
 177                                 rdy(NULL, pass, URL_READY_ERROR);
 178                                 break;
 179                 }
 180         }
 181         else
 182         {
 183                 float fh;
 184                 fh = fopen(url, mode);
 185                 if (fh < 0)
 186                 {
 187                         rdy(NULL, pass, URL_READY_ERROR);
 188                         return;
 189                 }
 190                 else
 191                 {
 192                         e = spawn();
 193                         e.classname = "url_single_fopen_file";
 194                         e.url_fh = fh;
 195                         e.url_ready = rdy;
 196                         e.url_ready_pass = pass;
 197                         if (mode == FILE_READ) rdy(e, pass, URL_READY_CANREAD);
 198                         else rdy(e, pass, URL_READY_CANWRITE);
 199                 }
 200         }
 201 }
 202
 203 // close a file
 204 void url_fclose(entity e)
 205 {
 206         int i;
 207
 208         if (e.url_fh == URL_FH_CURL)
 209         {
 210                 if (e.url_rbuf == -1 || e.url_wbuf != -1)     // not(post GET/POST request)
 211                         if (e.url_rbuf != -1 || e.url_wbuf == -1) // not(pre POST request)
 212                                 error("url_fclose: not closable in current state");
 213
 214                 // closing an URL!
 215                 if (e.url_wbuf >= 0)
 216                 {
 217                         // we are closing the write end (HTTP POST request)
 218
 219                         // get slot for HTTP request
 220                         for (i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 221                                 if (url_fromid[i] == NULL) break;
 222                         if (i >= NUM_URL_ID)
 223                         {
 224                                 for (i = 0; i < autocvar__urllib_nextslot; ++i)
 225                                         if (url_fromid[i] == NULL) break;
 226                                 if (i >= autocvar__urllib_nextslot)
 227                                 {
 228                                         LOG_INFO("url_fclose: too many concurrent requests\n");
 229                                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 230                                         buf_del(e.url_wbuf);
 231                                         strunzone(e.url_url);
 232                                         remove(e);
 233                                         return;
 234                                 }
 235                         }
 236
 237                         // POST the data
 238                         if (!crypto_uri_postbuf(e.url_url, i + MIN_URL_ID, "text/plain", "", e.url_wbuf, 0))
 239                         {
 240                                 LOG_INFO("url_fclose: failure in crypto_uri_postbuf\n");
 241                                 e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 242                                 buf_del(e.url_wbuf);
 243                                 strunzone(e.url_url);
 244                                 remove(e);
 245                                 return;
 246                         }
 247
 248                         // delete write end. File handle is now in unusable
 249                         // state. Wait for data to come from the server, then
 250                         // call the callback
 251                         buf_del(e.url_wbuf);
 252                         e.url_wbuf = -1;
 253                         e.url_id = i;
 254                         url_fromid[i] = e;
 255
 256                         // make sure this slot won't be reused quickly even on map change
 257                         cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 258                 }
 259                 else
 260                 {
 261                         // we have READ all data, just close
 262                         e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);
 263                         buf_del(e.url_rbuf);
 264                         strunzone(e.url_url);
 265                         remove(e);
 266                 }
 267         }
 268         else if (e.url_fh == URL_FH_STDOUT)
 269         {
 270                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);  // closing creates no reading handle
 271                 remove(e);
 272         }
 273         else
 274         {
 275                 // file
 276                 fclose(e.url_fh);
 277                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);  // closing creates no reading handle
 278                 remove(e);
 279         }
 280 }
 281
 282 // with \n (blame FRIK_FILE)
 283 string url_fgets(entity e)
 284 {
 285         if (e.url_fh == URL_FH_CURL)
 286         {
 287                 if (e.url_rbuf == -1) error("url_fgets: not readable in current state");
 288                 // curl
 289                 string s;
 290                 s = bufstr_get(e.url_rbuf, e.url_rbufpos);
 291                 e.url_rbufpos += 1;
 292                 return s;
 293         }
 294         else if (e.url_fh == URL_FH_STDOUT)
 295         {
 296                 // stdout
 297                 return string_null;
 298         }
 299         else
 300         {
 301                 // file
 302                 return fgets(e.url_fh);
 303         }
 304 }
 305
 306 // without \n (blame FRIK_FILE)
 307 void url_fputs(entity e, string s)
 308 {
 309         if (e.url_fh == URL_FH_CURL)
 310         {
 311                 if (e.url_wbuf == -1) error("url_fputs: not writable in current state");
 312                 // curl
 313                 bufstr_set(e.url_wbuf, e.url_wbufpos, s);
 314                 e.url_wbufpos += 1;
 315         }
 316         else if (e.url_fh == URL_FH_STDOUT)
 317         {
 318                 // stdout
 319                 LOG_INFO(s);
 320         }
 321         else
 322         {
 323                 // file
 324                 fputs(e.url_fh, s);
 325         }
 326 }
 327
 328 // multi URL object, tries URLs separated by space in sequence
 329 void url_multi_ready(entity fh, entity me, float status)
 330 {
 331         float n;
 332         if (status == URL_READY_ERROR || status < 0)
 333         {
 334                 if (status == -422)  // Unprocessable Entity
 335                 {
 336                         LOG_INFO("uri_multi_ready: got HTTP error 422, data is in unusable format - not continuing\n");
 337                         me.url_ready(fh, me.url_ready_pass, status);
 338                         strunzone(me.url_url);
 339                         remove(me);
 340                         return;
 341                 }
 342                 me.url_attempt += 1;
 343                 n = tokenize_console(me.url_url);
 344                 if (n <= me.url_attempt)
 345                 {
 346                         me.url_ready(fh, me.url_ready_pass, status);
 347                         strunzone(me.url_url);
 348                         remove(me);
 349                         return;
 350                 }
 351                 url_single_fopen(argv(me.url_attempt), me.url_mode, url_multi_ready, me);
 352                 return;
 353         }
 354         me.url_ready(fh, me.url_ready_pass, status);
 355 }
 356 void url_multi_fopen(string url, int mode, url_ready_func rdy, entity pass)
 357 {
 358         float n;
 359         n = tokenize_console(url);
 360         if (n <= 0)
 361         {
 362                 LOG_INFO("url_multi_fopen: need at least one URL\n");
 363                 rdy(NULL, pass, URL_READY_ERROR);
 364                 return;
 365         }
 366
 367         entity me;
 368         me = spawn();
 369         me.classname = "url_multi";
 370         me.url_url = strzone(url);
 371         me.url_attempt = 0;
 372         me.url_mode = mode;
 373         me.url_ready = rdy;
 374         me.url_ready_pass = pass;
 375         url_single_fopen(argv(0), mode, url_multi_ready, me);
 376 }