qcsrc/common/urllib.qc

   1 #if defined(CSQC)
   2         #include "../dpdefs/csprogsdefs.qh"
   3         #include "constants.qh"
   4         #include "util.qh"
   5         #include "urllib.qh"
   6 #elif defined(MENUQC)
   7 #elif defined(SVQC)
   8         #include "../dpdefs/progsdefs.qh"
   9     #include "../dpdefs/dpextensions.qh"
  10     #include "constants.qh"
  11     #include "util.qh"
  12     #include "urllib.qh"
  13 #endif
  14
  15 // files
  16 .float url_fh;
  17 const float URL_FH_CURL = -1;
  18 const float URL_FH_STDOUT = -2;
  19
  20 // URLs
  21 .string url_url;
  22 .float url_wbuf;
  23 .float url_wbufpos;
  24 .float url_rbuf;
  25 .float url_rbufpos;
  26 .float url_id;
  27 .url_ready_func url_ready;
  28 .entity url_ready_pass;
  29
  30 // for multi handles
  31 .int url_attempt;
  32 .int url_mode;
  33
  34 entity url_fromid[NUM_URL_ID];
  35 int autocvar__urllib_nextslot;
  36
  37 float url_URI_Get_Callback(int id, float status, string data)
  38 {
  39         if(id < MIN_URL_ID)
  40                 return 0;
  41         id -= MIN_URL_ID;
  42         if(id >= NUM_URL_ID)
  43                 return 0;
  44         entity e;
  45         e = url_fromid[id];
  46         if(!e)
  47                 return 0;
  48         if(e.url_rbuf >= 0 || e.url_wbuf >= 0)
  49         {
  50                 printf("WARNING: handle %d (%s) has already received data?!?\n", id + NUM_URL_ID, e.url_url);
  51                 return 0;
  52         }
  53
  54         // whatever happens, we will remove the URL from the list of IDs
  55         url_fromid[id] = world;
  56
  57         // if we get here, we MUST have both buffers cleared
  58         if(e.url_rbuf != -1 || e.url_wbuf != -1 || e.url_fh != URL_FH_CURL)
  59                 error("url_URI_Get_Callback: not a request waiting for data");
  60
  61         if(status == 0)
  62         {
  63                 // WE GOT DATA!
  64                 float n, i;
  65                 n = tokenizebyseparator(data, "\n");
  66                 e.url_rbuf = buf_create();
  67                 if(e.url_rbuf < 0)
  68                 {
  69                         print("url_URI_Get_Callback: out of memory in buf_create\n");
  70                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  71                         strunzone(e.url_url);
  72                         remove(e);
  73                         return 1;
  74                 }
  75                 e.url_rbufpos = 0;
  76                 if(e.url_rbuf < 0)
  77                 {
  78                         print("url_URI_Get_Callback: out of memory in buf_create\n");
  79                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  80                         strunzone(e.url_url);
  81                         remove(e);
  82                         return 1;
  83                 }
  84                 for(i = 0; i < n; ++i)
  85                         bufstr_set(e.url_rbuf, i, argv(i));
  86                 e.url_ready(e, e.url_ready_pass, URL_READY_CANREAD);
  87                 return 1;
  88         }
  89         else
  90         {
  91                 // an ERROR
  92                 e.url_ready(e, e.url_ready_pass, -fabs(status));
  93                 strunzone(e.url_url);
  94                 remove(e);
  95                 return 1;
  96         }
  97 }
  98
  99 void url_single_fopen(string url, int mode, url_ready_func rdy, entity pass)
 100 {
 101         entity e;
 102         int i;
 103         if(strstrofs(url, "://", 0) >= 0)
 104         {
 105                 switch(mode)
 106                 {
 107                         case FILE_WRITE:
 108                         case FILE_APPEND:
 109                                 // collect data to a stringbuffer for a POST request
 110                                 // attempts to close will result in a reading handle
 111
 112                                 // create a writing end that does nothing yet
 113                                 e = spawn();
 114                                 e.classname = "url_single_fopen_file";
 115                                 e.url_url = strzone(url);
 116                                 e.url_fh = URL_FH_CURL;
 117                                 e.url_wbuf = buf_create();
 118                                 if(e.url_wbuf < 0)
 119                                 {
 120                                         print("url_single_fopen: out of memory in buf_create\n");
 121                                         rdy(e, pass, URL_READY_ERROR);
 122                                         strunzone(e.url_url);
 123                                         remove(e);
 124                                         return;
 125                                 }
 126                                 e.url_wbufpos = 0;
 127                                 e.url_rbuf = -1;
 128                                 e.url_ready = rdy;
 129                                 e.url_ready_pass = pass;
 130                                 rdy(e, pass, URL_READY_CANWRITE);
 131                                 break;
 132
 133                         case FILE_READ:
 134                                 // read data only
 135
 136                                 // get slot for HTTP request
 137                                 for(i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 138                                         if(url_fromid[i] == world)
 139                                                 break;
 140                                 if(i >= NUM_URL_ID)
 141                                 {
 142                                         for(i = 0; i < autocvar__urllib_nextslot; ++i)
 143                                                 if(url_fromid[i] == world)
 144                                                         break;
 145                                         if(i >= autocvar__urllib_nextslot)
 146                                         {
 147                                                 print("url_single_fopen: too many concurrent requests\n");
 148                                                 rdy(world, pass, URL_READY_ERROR);
 149                                                 return;
 150                                         }
 151                                 }
 152
 153                                 // GET the data
 154                                 if(!crypto_uri_postbuf(url, i + MIN_URL_ID, string_null, string_null, -1, 0))
 155                                 {
 156                                         print("url_single_fopen: failure in crypto_uri_postbuf\n");
 157                                         rdy(world, pass, URL_READY_ERROR);
 158                                         return;
 159                                 }
 160
 161                                 // Make a dummy handle object (no buffers at
 162                                 // all). Wait for data to come from the
 163                                 // server, then call the callback
 164                                 e = spawn();
 165                                 e.classname = "url_single_fopen_file";
 166                                 e.url_url = strzone(url);
 167                                 e.url_fh = URL_FH_CURL;
 168                                 e.url_rbuf = -1;
 169                                 e.url_wbuf = -1;
 170                                 e.url_ready = rdy;
 171                                 e.url_ready_pass = pass;
 172                                 e.url_id = i;
 173                                 url_fromid[i] = e;
 174
 175                                 // make sure this slot won't be reused quickly even on map change
 176                                 cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 177                                 break;
 178                 }
 179         }
 180         else if(url == "-")
 181         {
 182                 switch(mode)
 183                 {
 184                         case FILE_WRITE:
 185                         case FILE_APPEND:
 186                                 e = spawn();
 187                                 e.classname = "url_single_fopen_stdout";
 188                                 e.url_fh = URL_FH_STDOUT;
 189                                 e.url_ready = rdy;
 190                                 e.url_ready_pass = pass;
 191                                 rdy(e, pass, URL_READY_CANWRITE);
 192                                 break;
 193                         case FILE_READ:
 194                                 print("url_single_fopen: cannot open '-' for reading\n");
 195                                 rdy(world, pass, URL_READY_ERROR);
 196                                 break;
 197                 }
 198         }
 199         else
 200         {
 201                 float fh;
 202                 fh = fopen(url, mode);
 203                 if(fh < 0)
 204                 {
 205                         rdy(world, pass, URL_READY_ERROR);
 206                         return;
 207                 }
 208                 else
 209                 {
 210                         e = spawn();
 211                         e.classname = "url_single_fopen_file";
 212                         e.url_fh = fh;
 213                         e.url_ready = rdy;
 214                         e.url_ready_pass = pass;
 215                         if(mode == FILE_READ)
 216                                 rdy(e, pass, URL_READY_CANREAD);
 217                         else
 218                                 rdy(e, pass, URL_READY_CANWRITE);
 219                 }
 220         }
 221 }
 222
 223 // close a file
 224 void url_fclose(entity e)
 225 {
 226         int i;
 227
 228         if(e.url_fh == URL_FH_CURL)
 229         {
 230                 if(e.url_rbuf == -1 || e.url_wbuf != -1) // not(post GET/POST request)
 231                 if(e.url_rbuf != -1 || e.url_wbuf == -1) // not(pre POST request)
 232                         error("url_fclose: not closable in current state");
 233
 234                 // closing an URL!
 235                 if(e.url_wbuf >= 0)
 236                 {
 237                         // we are closing the write end (HTTP POST request)
 238
 239                         // get slot for HTTP request
 240                         for(i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 241                                 if(url_fromid[i] == world)
 242                                         break;
 243                         if(i >= NUM_URL_ID)
 244                         {
 245                                 for(i = 0; i < autocvar__urllib_nextslot; ++i)
 246                                         if(url_fromid[i] == world)
 247                                                 break;
 248                                 if(i >= autocvar__urllib_nextslot)
 249                                 {
 250                                         print("url_fclose: too many concurrent requests\n");
 251                                         e.url_ready(e,e.url_ready_pass, URL_READY_ERROR);
 252                                         buf_del(e.url_wbuf);
 253                                         strunzone(e.url_url);
 254                                         remove(e);
 255                                         return;
 256                                 }
 257                         }
 258
 259                         // POST the data
 260                         if(!crypto_uri_postbuf(e.url_url, i + MIN_URL_ID, "text/plain", "", e.url_wbuf, 0))
 261                         {
 262                                 print("url_fclose: failure in crypto_uri_postbuf\n");
 263                                 e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 264                                 buf_del(e.url_wbuf);
 265                                 strunzone(e.url_url);
 266                                 remove(e);
 267                                 return;
 268                         }
 269
 270                         // delete write end. File handle is now in unusable
 271                         // state. Wait for data to come from the server, then
 272                         // call the callback
 273                         buf_del(e.url_wbuf);
 274                         e.url_wbuf = -1;
 275                         e.url_id = i;
 276                         url_fromid[i] = e;
 277
 278                         // make sure this slot won't be reused quickly even on map change
 279                         cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 280                 }
 281                 else
 282                 {
 283                         // we have READ all data, just close
 284                         e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);
 285                         buf_del(e.url_rbuf);
 286                         strunzone(e.url_url);
 287                         remove(e);
 288                 }
 289         }
 290         else if(e.url_fh == URL_FH_STDOUT)
 291         {
 292                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED); // closing creates no reading handle
 293                 remove(e);
 294         }
 295         else
 296         {
 297                 // file
 298                 fclose(e.url_fh);
 299                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED); // closing creates no reading handle
 300                 remove(e);
 301         }
 302 }
 303
 304 // with \n (blame FRIK_FILE)
 305 string url_fgets(entity e)
 306 {
 307         if(e.url_fh == URL_FH_CURL)
 308         {
 309                 if(e.url_rbuf == -1)
 310                         error("url_fgets: not readable in current state");
 311                 // curl
 312                 string s;
 313                 s = bufstr_get(e.url_rbuf, e.url_rbufpos);
 314                 e.url_rbufpos += 1;
 315                 return s;
 316         }
 317         else if(e.url_fh == URL_FH_STDOUT)
 318         {
 319                 // stdout
 320                 return string_null;
 321         }
 322         else
 323         {
 324                 // file
 325                 return fgets(e.url_fh);
 326         }
 327 }
 328
 329 // without \n (blame FRIK_FILE)
 330 void url_fputs(entity e, string s)
 331 {
 332         if(e.url_fh == URL_FH_CURL)
 333         {
 334                 if(e.url_wbuf == -1)
 335                         error("url_fputs: not writable in current state");
 336                 // curl
 337                 bufstr_set(e.url_wbuf, e.url_wbufpos, s);
 338                 e.url_wbufpos += 1;
 339         }
 340         else if(e.url_fh == URL_FH_STDOUT)
 341         {
 342                 // stdout
 343                 print(s);
 344         }
 345         else
 346         {
 347                 // file
 348                 fputs(e.url_fh, s);
 349         }
 350 }
 351
 352 // multi URL object, tries URLs separated by space in sequence
 353 void url_multi_ready(entity fh, entity me, float status)
 354 {
 355         float n;
 356         if(status == URL_READY_ERROR || status < 0)
 357         {
 358                 if(status == -422) // Unprocessable Entity
 359                 {
 360                         print("uri_multi_ready: got HTTP error 422, data is in unusable format - not continuing\n");
 361                         me.url_ready(fh, me.url_ready_pass, status);
 362                         strunzone(me.url_url);
 363                         remove(me);
 364                         return;
 365                 }
 366                 me.url_attempt += 1;
 367                 n = tokenize_console(me.url_url);
 368                 if(n <= me.url_attempt)
 369                 {
 370                         me.url_ready(fh, me.url_ready_pass, status);
 371                         strunzone(me.url_url);
 372                         remove(me);
 373                         return;
 374                 }
 375                 url_single_fopen(argv(me.url_attempt), me.url_mode, url_multi_ready, me);
 376                 return;
 377         }
 378         me.url_ready(fh, me.url_ready_pass, status);
 379 }
 380 void url_multi_fopen(string url, int mode, url_ready_func rdy, entity pass)
 381 {
 382         float n;
 383         n = tokenize_console(url);
 384         if(n <= 0)
 385         {
 386                 print("url_multi_fopen: need at least one URL\n");
 387                 rdy(world, pass, URL_READY_ERROR);
 388                 return;
 389         }
 390
 391         entity me;
 392         me = spawn();
 393         me.classname = "url_multi";
 394         me.url_url = strzone(url);
 395         me.url_attempt = 0;
 396         me.url_mode = mode;
 397         me.url_ready = rdy;
 398         me.url_ready_pass = pass;
 399         url_single_fopen(argv(0), mode, url_multi_ready, me);
 400 }