qcsrc/common/urllib.qc

   1 #if defined(CSQC)
   2         #include "../dpdefs/csprogsdefs.qc"
   3         #include "constants.qh"
   4         #include "util.qh"
   5         #include "urllib.qh"
   6 #elif defined(MENUQC)
   7 #elif defined(SVQC)
   8         #include "../dpdefs/progsdefs.qc"
   9     #include "../dpdefs/dpextensions.qc"
  10     #include "../server/sys-post.qh"
  11     #include "constants.qh"
  12     #include "util.qh"
  13     #include "urllib.qh"
  14 #endif
  15
  16 // files
  17 .float url_fh;
  18 const float URL_FH_CURL = -1;
  19 const float URL_FH_STDOUT = -2;
  20
  21 // URLs
  22 .string url_url;
  23 .float url_wbuf;
  24 .float url_wbufpos;
  25 .float url_rbuf;
  26 .float url_rbufpos;
  27 .float url_id;
  28 .url_ready_func url_ready;
  29 .entity url_ready_pass;
  30
  31 // for multi handles
  32 .float url_attempt;
  33 .float url_mode;
  34
  35 entity url_fromid[NUM_URL_ID];
  36 float autocvar__urllib_nextslot;
  37
  38 float url_URI_Get_Callback(float id, float status, string data)
  39 {
  40         if(id < MIN_URL_ID)
  41                 return 0;
  42         id -= MIN_URL_ID;
  43         if(id >= NUM_URL_ID)
  44                 return 0;
  45         entity e;
  46         e = url_fromid[id];
  47         if(!e)
  48                 return 0;
  49         if(e.url_rbuf >= 0 || e.url_wbuf >= 0)
  50         {
  51                 printf("WARNING: handle %d (%s) has already received data?!?\n", id + NUM_URL_ID, e.url_url);
  52                 return 0;
  53         }
  54
  55         // whatever happens, we will remove the URL from the list of IDs
  56         url_fromid[id] = world;
  57
  58         // if we get here, we MUST have both buffers cleared
  59         if(e.url_rbuf != -1 || e.url_wbuf != -1 || e.url_fh != URL_FH_CURL)
  60                 error("url_URI_Get_Callback: not a request waiting for data");
  61
  62         if(status == 0)
  63         {
  64                 // WE GOT DATA!
  65                 float n, i;
  66                 n = tokenizebyseparator(data, "\n");
  67                 e.url_rbuf = buf_create();
  68                 if(e.url_rbuf < 0)
  69                 {
  70                         print("url_URI_Get_Callback: out of memory in buf_create\n");
  71                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  72                         strunzone(e.url_url);
  73                         remove(e);
  74                         return 1;
  75                 }
  76                 e.url_rbufpos = 0;
  77                 if(e.url_rbuf < 0)
  78                 {
  79                         print("url_URI_Get_Callback: out of memory in buf_create\n");
  80                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  81                         strunzone(e.url_url);
  82                         remove(e);
  83                         return 1;
  84                 }
  85                 for(i = 0; i < n; ++i)
  86                         bufstr_set(e.url_rbuf, i, argv(i));
  87                 e.url_ready(e, e.url_ready_pass, URL_READY_CANREAD);
  88                 return 1;
  89         }
  90         else
  91         {
  92                 // an ERROR
  93                 e.url_ready(e, e.url_ready_pass, -fabs(status));
  94                 strunzone(e.url_url);
  95                 remove(e);
  96                 return 1;
  97         }
  98 }
  99
 100 void url_single_fopen(string url, float mode, url_ready_func rdy, entity pass)
 101 {
 102         entity e;
 103         int i;
 104         if(strstrofs(url, "://", 0) >= 0)
 105         {
 106                 switch(mode)
 107                 {
 108                         case FILE_WRITE:
 109                         case FILE_APPEND:
 110                                 // collect data to a stringbuffer for a POST request
 111                                 // attempts to close will result in a reading handle
 112
 113                                 // create a writing end that does nothing yet
 114                                 e = spawn();
 115                                 e.classname = "url_single_fopen_file";
 116                                 e.url_url = strzone(url);
 117                                 e.url_fh = URL_FH_CURL;
 118                                 e.url_wbuf = buf_create();
 119                                 if(e.url_wbuf < 0)
 120                                 {
 121                                         print("url_single_fopen: out of memory in buf_create\n");
 122                                         rdy(e, pass, URL_READY_ERROR);
 123                                         strunzone(e.url_url);
 124                                         remove(e);
 125                                         return;
 126                                 }
 127                                 e.url_wbufpos = 0;
 128                                 e.url_rbuf = -1;
 129                                 e.url_ready = rdy;
 130                                 e.url_ready_pass = pass;
 131                                 rdy(e, pass, URL_READY_CANWRITE);
 132                                 break;
 133
 134                         case FILE_READ:
 135                                 // read data only
 136
 137                                 // get slot for HTTP request
 138                                 for(i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 139                                         if(url_fromid[i] == world)
 140                                                 break;
 141                                 if(i >= NUM_URL_ID)
 142                                 {
 143                                         for(i = 0; i < autocvar__urllib_nextslot; ++i)
 144                                                 if(url_fromid[i] == world)
 145                                                         break;
 146                                         if(i >= autocvar__urllib_nextslot)
 147                                         {
 148                                                 print("url_single_fopen: too many concurrent requests\n");
 149                                                 rdy(world, pass, URL_READY_ERROR);
 150                                                 return;
 151                                         }
 152                                 }
 153
 154                                 // GET the data
 155                                 if(!crypto_uri_postbuf(url, i + MIN_URL_ID, string_null, string_null, -1, 0))
 156                                 {
 157                                         print("url_single_fopen: failure in crypto_uri_postbuf\n");
 158                                         rdy(world, pass, URL_READY_ERROR);
 159                                         return;
 160                                 }
 161
 162                                 // Make a dummy handle object (no buffers at
 163                                 // all). Wait for data to come from the
 164                                 // server, then call the callback
 165                                 e = spawn();
 166                                 e.classname = "url_single_fopen_file";
 167                                 e.url_url = strzone(url);
 168                                 e.url_fh = URL_FH_CURL;
 169                                 e.url_rbuf = -1;
 170                                 e.url_wbuf = -1;
 171                                 e.url_ready = rdy;
 172                                 e.url_ready_pass = pass;
 173                                 e.url_id = i;
 174                                 url_fromid[i] = e;
 175
 176                                 // make sure this slot won't be reused quickly even on map change
 177                                 cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 178                                 break;
 179                 }
 180         }
 181         else if(url == "-")
 182         {
 183                 switch(mode)
 184                 {
 185                         case FILE_WRITE:
 186                         case FILE_APPEND:
 187                                 e = spawn();
 188                                 e.classname = "url_single_fopen_stdout";
 189                                 e.url_fh = URL_FH_STDOUT;
 190                                 e.url_ready = rdy;
 191                                 e.url_ready_pass = pass;
 192                                 rdy(e, pass, URL_READY_CANWRITE);
 193                                 break;
 194                         case FILE_READ:
 195                                 print("url_single_fopen: cannot open '-' for reading\n");
 196                                 rdy(world, pass, URL_READY_ERROR);
 197                                 break;
 198                 }
 199         }
 200         else
 201         {
 202                 float fh;
 203                 fh = fopen(url, mode);
 204                 if(fh < 0)
 205                 {
 206                         rdy(world, pass, URL_READY_ERROR);
 207                         return;
 208                 }
 209                 else
 210                 {
 211                         e = spawn();
 212                         e.classname = "url_single_fopen_file";
 213                         e.url_fh = fh;
 214                         e.url_ready = rdy;
 215                         e.url_ready_pass = pass;
 216                         if(mode == FILE_READ)
 217                                 rdy(e, pass, URL_READY_CANREAD);
 218                         else
 219                                 rdy(e, pass, URL_READY_CANWRITE);
 220                 }
 221         }
 222 }
 223
 224 // close a file
 225 void url_fclose(entity e)
 226 {
 227         int i;
 228
 229         if(e.url_fh == URL_FH_CURL)
 230         {
 231                 if(e.url_rbuf == -1 || e.url_wbuf != -1) // not(post GET/POST request)
 232                 if(e.url_rbuf != -1 || e.url_wbuf == -1) // not(pre POST request)
 233                         error("url_fclose: not closable in current state");
 234
 235                 // closing an URL!
 236                 if(e.url_wbuf >= 0)
 237                 {
 238                         // we are closing the write end (HTTP POST request)
 239
 240                         // get slot for HTTP request
 241                         for(i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 242                                 if(url_fromid[i] == world)
 243                                         break;
 244                         if(i >= NUM_URL_ID)
 245                         {
 246                                 for(i = 0; i < autocvar__urllib_nextslot; ++i)
 247                                         if(url_fromid[i] == world)
 248                                                 break;
 249                                 if(i >= autocvar__urllib_nextslot)
 250                                 {
 251                                         print("url_fclose: too many concurrent requests\n");
 252                                         e.url_ready(e,e.url_ready_pass, URL_READY_ERROR);
 253                                         buf_del(e.url_wbuf);
 254                                         strunzone(e.url_url);
 255                                         remove(e);
 256                                         return;
 257                                 }
 258                         }
 259
 260                         // POST the data
 261                         if(!crypto_uri_postbuf(e.url_url, i + MIN_URL_ID, "text/plain", "", e.url_wbuf, 0))
 262                         {
 263                                 print("url_fclose: failure in crypto_uri_postbuf\n");
 264                                 e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 265                                 buf_del(e.url_wbuf);
 266                                 strunzone(e.url_url);
 267                                 remove(e);
 268                                 return;
 269                         }
 270
 271                         // delete write end. File handle is now in unusable
 272                         // state. Wait for data to come from the server, then
 273                         // call the callback
 274                         buf_del(e.url_wbuf);
 275                         e.url_wbuf = -1;
 276                         e.url_id = i;
 277                         url_fromid[i] = e;
 278
 279                         // make sure this slot won't be reused quickly even on map change
 280                         cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 281                 }
 282                 else
 283                 {
 284                         // we have READ all data, just close
 285                         e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);
 286                         buf_del(e.url_rbuf);
 287                         strunzone(e.url_url);
 288                         remove(e);
 289                 }
 290         }
 291         else if(e.url_fh == URL_FH_STDOUT)
 292         {
 293                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED); // closing creates no reading handle
 294                 remove(e);
 295         }
 296         else
 297         {
 298                 // file
 299                 fclose(e.url_fh);
 300                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED); // closing creates no reading handle
 301                 remove(e);
 302         }
 303 }
 304
 305 // with \n (blame FRIK_FILE)
 306 string url_fgets(entity e)
 307 {
 308         if(e.url_fh == URL_FH_CURL)
 309         {
 310                 if(e.url_rbuf == -1)
 311                         error("url_fgets: not readable in current state");
 312                 // curl
 313                 string s;
 314                 s = bufstr_get(e.url_rbuf, e.url_rbufpos);
 315                 e.url_rbufpos += 1;
 316                 return s;
 317         }
 318         else if(e.url_fh == URL_FH_STDOUT)
 319         {
 320                 // stdout
 321                 return string_null;
 322         }
 323         else
 324         {
 325                 // file
 326                 return fgets(e.url_fh);
 327         }
 328 }
 329
 330 // without \n (blame FRIK_FILE)
 331 void url_fputs(entity e, string s)
 332 {
 333         if(e.url_fh == URL_FH_CURL)
 334         {
 335                 if(e.url_wbuf == -1)
 336                         error("url_fputs: not writable in current state");
 337                 // curl
 338                 bufstr_set(e.url_wbuf, e.url_wbufpos, s);
 339                 e.url_wbufpos += 1;
 340         }
 341         else if(e.url_fh == URL_FH_STDOUT)
 342         {
 343                 // stdout
 344                 print(s);
 345         }
 346         else
 347         {
 348                 // file
 349                 fputs(e.url_fh, s);
 350         }
 351 }
 352
 353 // multi URL object, tries URLs separated by space in sequence
 354 void url_multi_ready(entity fh, entity me, float status)
 355 {
 356         float n;
 357         if(status == URL_READY_ERROR || status < 0)
 358         {
 359                 if(status == -422) // Unprocessable Entity
 360                 {
 361                         print("uri_multi_ready: got HTTP error 422, data is in unusable format - not continuing\n");
 362                         me.url_ready(fh, me.url_ready_pass, status);
 363                         strunzone(me.url_url);
 364                         remove(me);
 365                         return;
 366                 }
 367                 me.url_attempt += 1;
 368                 n = tokenize_console(me.url_url);
 369                 if(n <= me.url_attempt)
 370                 {
 371                         me.url_ready(fh, me.url_ready_pass, status);
 372                         strunzone(me.url_url);
 373                         remove(me);
 374                         return;
 375                 }
 376                 url_single_fopen(argv(me.url_attempt), me.url_mode, url_multi_ready, me);
 377                 return;
 378         }
 379         me.url_ready(fh, me.url_ready_pass, status);
 380 }
 381 void url_multi_fopen(string url, float mode, url_ready_func rdy, entity pass)
 382 {
 383         float n;
 384         n = tokenize_console(url);
 385         if(n <= 0)
 386         {
 387                 print("url_multi_fopen: need at least one URL\n");
 388                 rdy(world, pass, URL_READY_ERROR);
 389                 return;
 390         }
 391
 392         entity me;
 393         me = spawn();
 394         me.classname = "url_multi";
 395         me.url_url = strzone(url);
 396         me.url_attempt = 0;
 397         me.url_mode = mode;
 398         me.url_ready = rdy;
 399         me.url_ready_pass = pass;
 400         url_single_fopen(argv(0), mode, url_multi_ready, me);
 401 }