qcsrc/common/urllib.qc

   1 // files
   2 .float url_fh;
   3 #define URL_FH_CURL   -1
   4 #define URL_FH_STDOUT -2
   5
   6 // URLs
   7 .string url_url;
   8 .float url_wbuf;
   9 .float url_wbufpos;
  10 .float url_rbuf;
  11 .float url_rbufpos;
  12 .float url_id;
  13 .url_ready_func url_ready;
  14 .entity url_ready_pass;
  15
  16 // for multi handles
  17 .float url_attempt;
  18 .float url_mode;
  19
  20 entity url_fromid[NUM_URL_ID];
  21 float autocvar__urllib_nextslot;
  22
  23 float url_URI_Get_Callback(float id, float status, string data)
  24 {
  25         if(id < MIN_URL_ID)
  26                 return 0;
  27         id -= MIN_URL_ID;
  28         if(id >= NUM_URL_ID)
  29                 return 0;
  30         entity e;
  31         e = url_fromid[id];
  32         if(!e)
  33                 return 0;
  34         if(e.url_rbuf >= 0 || e.url_wbuf >= 0)
  35         {
  36                 print(sprintf("WARNING: handle %d (%s) has already received data?!?\n", id + NUM_URL_ID, e.url_url));
  37                 return 0;
  38         }
  39
  40         // whatever happens, we will remove the URL from the list of IDs
  41         url_fromid[id] = world;
  42
  43         // if we get here, we MUST have both buffers cleared
  44         if(e.url_rbuf != -1 || e.url_wbuf != -1 || e.url_fh != URL_FH_CURL)
  45                 error("url_URI_Get_Callback: not a request waiting for data");
  46
  47         if(status == 0)
  48         {
  49                 // WE GOT DATA!
  50                 float n, i;
  51                 n = tokenizebyseparator(data, "\n");
  52                 e.url_rbuf = buf_create();
  53                 if(e.url_rbuf < 0)
  54                 {
  55                         print("url_URI_Get_Callback: out of memory in buf_create\n");
  56                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  57                         strunzone(e.url_url);
  58                         remove(e);
  59                         return 1;
  60                 }
  61                 e.url_rbufpos = 0;
  62                 if(e.url_rbuf < 0)
  63                 {
  64                         print("url_URI_Get_Callback: out of memory in buf_create\n");
  65                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  66                         strunzone(e.url_url);
  67                         remove(e);
  68                         return 1;
  69                 }
  70                 for(i = 0; i < n; ++i)
  71                         bufstr_set(e.url_rbuf, i, argv(i));
  72                 e.url_ready(e, e.url_ready_pass, URL_READY_CANREAD);
  73                 return 1;
  74         }
  75         else
  76         {
  77                 // an ERROR
  78                 e.url_ready(e, e.url_ready_pass, -fabs(status));
  79                 strunzone(e.url_url);
  80                 remove(e);
  81                 return 1;
  82         }
  83 }
  84
  85 void url_single_fopen(string url, float mode, url_ready_func rdy, entity pass)
  86 {
  87         entity e;
  88         float i;
  89         if(strstrofs(url, "://", 0) >= 0)
  90         {
  91                 switch(mode)
  92                 {
  93                         case FILE_WRITE:
  94                         case FILE_APPEND:
  95                                 // collect data to a stringbuffer for a POST request
  96                                 // attempts to close will result in a reading handle
  97
  98                                 // create a writing end that does nothing yet
  99                                 e = spawn();
 100                                 e.classname = "url_single_fopen_file";
 101                                 e.url_url = strzone(url);
 102                                 e.url_fh = URL_FH_CURL;
 103                                 e.url_wbuf = buf_create();
 104                                 if(e.url_wbuf < 0)
 105                                 {
 106                                         print("url_single_fopen: out of memory in buf_create\n");
 107                                         rdy(e, pass, URL_READY_ERROR);
 108                                         strunzone(e.url_url);
 109                                         remove(e);
 110                                         return;
 111                                 }
 112                                 e.url_wbufpos = 0;
 113                                 e.url_rbuf = -1;
 114                                 e.url_ready = rdy;
 115                                 e.url_ready_pass = pass;
 116                                 rdy(e, pass, URL_READY_CANWRITE);
 117                                 break;
 118
 119                         case FILE_READ:
 120                                 // read data only
 121
 122                                 // get slot for HTTP request
 123                                 for(i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 124                                         if(url_fromid[i] == world)
 125                                                 break;
 126                                 if(i >= NUM_URL_ID)
 127                                 {
 128                                         for(i = 0; i < autocvar__urllib_nextslot; ++i)
 129                                                 if(url_fromid[i] == world)
 130                                                         break;
 131                                         if(i >= autocvar__urllib_nextslot)
 132                                         {
 133                                                 print("url_single_fopen: too many concurrent requests\n");
 134                                                 rdy(world, pass, URL_READY_ERROR);
 135                                                 return;
 136                                         }
 137                                 }
 138
 139                                 // GET the data
 140                                 if(!crypto_uri_postbuf(url, i + MIN_URL_ID, string_null, string_null, -1, 0))
 141                                 {
 142                                         print("url_single_fopen: failure in crypto_uri_postbuf\n");
 143                                         rdy(world, pass, URL_READY_ERROR);
 144                                         return;
 145                                 }
 146
 147                                 // Make a dummy handle object (no buffers at
 148                                 // all). Wait for data to come from the
 149                                 // server, then call the callback
 150                                 e = spawn();
 151                                 e.classname = "url_single_fopen_file";
 152                                 e.url_url = strzone(url);
 153                                 e.url_fh = URL_FH_CURL;
 154                                 e.url_rbuf = -1;
 155                                 e.url_wbuf = -1;
 156                                 e.url_ready = rdy;
 157                                 e.url_ready_pass = pass;
 158                                 e.url_id = i;
 159                                 url_fromid[i] = e;
 160
 161                                 // make sure this slot won't be reused quickly even on map change
 162                                 cvar_set("_urllib_nextslot", ftos(mod(i + 1, NUM_URL_ID)));
 163                                 break;
 164                 }
 165         }
 166         else if(url == "-")
 167         {
 168                 switch(mode)
 169                 {
 170                         case FILE_WRITE:
 171                         case FILE_APPEND:
 172                                 e = spawn();
 173                                 e.classname = "url_single_fopen_stdout";
 174                                 e.url_fh = URL_FH_STDOUT;
 175                                 e.url_ready = rdy;
 176                                 e.url_ready_pass = pass;
 177                                 rdy(e, pass, URL_READY_CANWRITE);
 178                                 break;
 179                         case FILE_READ:
 180                                 print("url_single_fopen: cannot open '-' for reading\n");
 181                                 rdy(world, pass, URL_READY_ERROR);
 182                                 break;
 183                 }
 184         }
 185         else
 186         {
 187                 float fh;
 188                 fh = fopen(url, mode);
 189                 if(fh < 0)
 190                 {
 191                         rdy(world, pass, URL_READY_ERROR);
 192                         return;
 193                 }
 194                 else
 195                 {
 196                         e = spawn();
 197                         e.classname = "url_single_fopen_file";
 198                         e.url_fh = fh;
 199                         e.url_ready = rdy;
 200                         e.url_ready_pass = pass;
 201                         if(mode == FILE_READ)
 202                                 rdy(e, pass, URL_READY_CANREAD);
 203                         else
 204                                 rdy(e, pass, URL_READY_CANWRITE);
 205                 }
 206         }
 207 }
 208
 209 // close a file
 210 void url_fclose(entity e)
 211 {
 212         float i;
 213
 214         if(e.url_fh == URL_FH_CURL)
 215         {
 216                 if(e.url_rbuf == -1 || e.url_wbuf != -1) // not(post GET/POST request)
 217                 if(e.url_rbuf != -1 || e.url_wbuf == -1) // not(pre POST request)
 218                         error("url_fclose: not closable in current state");
 219
 220                 // closing an URL!
 221                 if(e.url_wbuf >= 0)
 222                 {
 223                         // we are closing the write end (HTTP POST request)
 224
 225                         // get slot for HTTP request
 226                         for(i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 227                                 if(url_fromid[i] == world)
 228                                         break;
 229                         if(i >= NUM_URL_ID)
 230                         {
 231                                 for(i = 0; i < autocvar__urllib_nextslot; ++i)
 232                                         if(url_fromid[i] == world)
 233                                                 break;
 234                                 if(i >= autocvar__urllib_nextslot)
 235                                 {
 236                                         print("url_fclose: too many concurrent requests\n");
 237                                         e.url_ready(e,e.url_ready_pass, URL_READY_ERROR);
 238                                         buf_del(e.url_wbuf);
 239                                         strunzone(e.url_url);
 240                                         remove(e);
 241                                         return;
 242                                 }
 243                         }
 244
 245                         // POST the data
 246                         if(!crypto_uri_postbuf(e.url_url, i + MIN_URL_ID, "text/plain", "", e.url_wbuf, 0))
 247                         {
 248                                 print("url_fclose: failure in crypto_uri_postbuf\n");
 249                                 e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 250                                 buf_del(e.url_wbuf);
 251                                 strunzone(e.url_url);
 252                                 remove(e);
 253                                 return;
 254                         }
 255
 256                         // delete write end. File handle is now in unusable
 257                         // state. Wait for data to come from the server, then
 258                         // call the callback
 259                         buf_del(e.url_wbuf);
 260                         e.url_wbuf = -1;
 261                         e.url_id = i;
 262                         url_fromid[i] = e;
 263
 264                         // make sure this slot won't be reused quickly even on map change
 265                         cvar_set("_urllib_nextslot", ftos(mod(i + 1, NUM_URL_ID)));
 266                 }
 267                 else
 268                 {
 269                         // we have READ all data, just close
 270                         e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);
 271                         buf_del(e.url_rbuf);
 272                         strunzone(e.url_url);
 273                         remove(e);
 274                 }
 275         }
 276         else if(e.url_fh == URL_FH_STDOUT)
 277         {
 278                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED); // closing creates no reading handle
 279                 remove(e);
 280         }
 281         else
 282         {
 283                 // file
 284                 fclose(e.url_fh);
 285                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED); // closing creates no reading handle
 286                 remove(e);
 287         }
 288 }
 289
 290 // with \n (blame FRIK_FILE)
 291 string url_fgets(entity e)
 292 {
 293         if(e.url_fh == URL_FH_CURL)
 294         {
 295                 if(e.url_rbuf == -1)
 296                         error("url_fgets: not readable in current state");
 297                 // curl
 298                 string s;
 299                 s = bufstr_get(e.url_rbuf, e.url_rbufpos);
 300                 e.url_rbufpos += 1;
 301                 return s;
 302         }
 303         else if(e.url_fh == URL_FH_STDOUT)
 304         {
 305                 // stdout
 306                 return string_null;
 307         }
 308         else
 309         {
 310                 // file
 311                 return fgets(e.url_fh);
 312         }
 313 }
 314
 315 // without \n (blame FRIK_FILE)
 316 void url_fputs(entity e, string s)
 317 {
 318         if(e.url_fh == URL_FH_CURL)
 319         {
 320                 if(e.url_wbuf == -1)
 321                         error("url_fputs: not writable in current state");
 322                 // curl
 323                 bufstr_set(e.url_wbuf, e.url_wbufpos, s);
 324                 e.url_wbufpos += 1;
 325         }
 326         else if(e.url_fh == URL_FH_STDOUT)
 327         {
 328                 // stdout
 329                 print(s);
 330         }
 331         else
 332         {
 333                 // file
 334                 fputs(e.url_fh, s);
 335         }
 336 }
 337
 338 // multi URL object, tries URLs separated by space in sequence
 339 void url_multi_ready(entity fh, entity me, float status)
 340 {
 341         float n;
 342         if(status == URL_READY_ERROR || status < 0)
 343         {
 344                 if(status == -422) // Unprocessable Entity
 345                 {
 346                         print("uri_multi_ready: got HTTP error 422, data is in unusable format - not continuing\n");
 347                         me.url_ready(fh, me.url_ready_pass, status);
 348                         strunzone(me.url_url);
 349                         remove(me);
 350                         return;
 351                 }
 352                 me.url_attempt += 1;
 353                 n = tokenize_console(me.url_url);
 354                 if(n <= me.url_attempt)
 355                 {
 356                         me.url_ready(fh, me.url_ready_pass, status);
 357                         strunzone(me.url_url);
 358                         remove(me);
 359                         return;
 360                 }
 361                 url_single_fopen(argv(me.url_attempt), me.url_mode, url_multi_ready, me);
 362                 return;
 363         }
 364         me.url_ready(fh, me.url_ready_pass, status);
 365 }
 366 void url_multi_fopen(string url, float mode, url_ready_func rdy, entity pass)
 367 {
 368         float n;
 369         n = tokenize_console(url);
 370         if(n <= 0)
 371         {
 372                 print("url_multi_fopen: need at least one URL\n");
 373                 rdy(world, pass, URL_READY_ERROR);
 374                 return;
 375         }
 376
 377         entity me;
 378         me = spawn();
 379         me.classname = "url_multi";
 380         me.url_url = strzone(url);
 381         me.url_attempt = 0;
 382         me.url_mode = mode;
 383         me.url_ready = rdy;
 384         me.url_ready_pass = pass;
 385         url_single_fopen(argv(0), mode, url_multi_ready, me);
 386 }