qcsrc/lib/urllib.qc

   1 #include "urllib.qh"
   2
   3 // files
   4 .float url_fh;
   5 const float URL_FH_CURL = -1;
   6 const float URL_FH_STDOUT = -2;
   7
   8 // URLs
   9 .string url_url;
  10 .float url_wbuf;
  11 .float url_wbufpos;
  12 .float url_rbuf;
  13 .float url_rbufpos;
  14 .float url_id;
  15 .url_ready_func url_ready;
  16 .entity url_ready_pass;
  17
  18 // for multi handles
  19 .int url_attempt;
  20 .int url_mode;
  21
  22 entity url_fromid[NUM_URL_ID];
  23 int autocvar__urllib_nextslot;
  24
  25 float url_URI_Get_Callback(int id, float status, string data)
  26 {
  27         if(id < MIN_URL_ID)
  28                 return 0;
  29         id -= MIN_URL_ID;
  30         if(id >= NUM_URL_ID)
  31                 return 0;
  32         entity e;
  33         e = url_fromid[id];
  34         if(!e)
  35                 return 0;
  36         if(e.url_rbuf >= 0 || e.url_wbuf >= 0)
  37         {
  38                 printf("WARNING: handle %d (%s) has already received data?!?\n", id + NUM_URL_ID, e.url_url);
  39                 return 0;
  40         }
  41
  42         // whatever happens, we will remove the URL from the list of IDs
  43         url_fromid[id] = NULL;
  44
  45         // if we get here, we MUST have both buffers cleared
  46         if(e.url_rbuf != -1 || e.url_wbuf != -1 || e.url_fh != URL_FH_CURL)
  47                 error("url_URI_Get_Callback: not a request waiting for data");
  48
  49         if(status == 0)
  50         {
  51                 // WE GOT DATA!
  52                 float n, i;
  53                 n = tokenizebyseparator(data, "\n");
  54                 e.url_rbuf = buf_create();
  55                 if(e.url_rbuf < 0)
  56                 {
  57                         print("url_URI_Get_Callback: out of memory in buf_create\n");
  58                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  59                         strunzone(e.url_url);
  60                         remove(e);
  61                         return 1;
  62                 }
  63                 e.url_rbufpos = 0;
  64                 if(e.url_rbuf < 0)
  65                 {
  66                         print("url_URI_Get_Callback: out of memory in buf_create\n");
  67                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  68                         strunzone(e.url_url);
  69                         remove(e);
  70                         return 1;
  71                 }
  72                 for(i = 0; i < n; ++i)
  73                         bufstr_set(e.url_rbuf, i, argv(i));
  74                 e.url_ready(e, e.url_ready_pass, URL_READY_CANREAD);
  75                 return 1;
  76         }
  77         else
  78         {
  79                 // an ERROR
  80                 e.url_ready(e, e.url_ready_pass, -fabs(status));
  81                 strunzone(e.url_url);
  82                 remove(e);
  83                 return 1;
  84         }
  85 }
  86
  87 void url_single_fopen(string url, int mode, url_ready_func rdy, entity pass)
  88 {
  89         entity e;
  90         int i;
  91         if(strstrofs(url, "://", 0) >= 0)
  92         {
  93                 switch(mode)
  94                 {
  95                         case FILE_WRITE:
  96                         case FILE_APPEND:
  97                                 // collect data to a stringbuffer for a POST request
  98                                 // attempts to close will result in a reading handle
  99
 100                                 // create a writing end that does nothing yet
 101                                 e = spawn();
 102                                 e.classname = "url_single_fopen_file";
 103                                 e.url_url = strzone(url);
 104                                 e.url_fh = URL_FH_CURL;
 105                                 e.url_wbuf = buf_create();
 106                                 if(e.url_wbuf < 0)
 107                                 {
 108                                         print("url_single_fopen: out of memory in buf_create\n");
 109                                         rdy(e, pass, URL_READY_ERROR);
 110                                         strunzone(e.url_url);
 111                                         remove(e);
 112                                         return;
 113                                 }
 114                                 e.url_wbufpos = 0;
 115                                 e.url_rbuf = -1;
 116                                 e.url_ready = rdy;
 117                                 e.url_ready_pass = pass;
 118                                 rdy(e, pass, URL_READY_CANWRITE);
 119                                 break;
 120
 121                         case FILE_READ:
 122                                 // read data only
 123
 124                                 // get slot for HTTP request
 125                                 for(i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 126                                         if(url_fromid[i] == NULL)
 127                                                 break;
 128                                 if(i >= NUM_URL_ID)
 129                                 {
 130                                         for(i = 0; i < autocvar__urllib_nextslot; ++i)
 131                                                 if(url_fromid[i] == NULL)
 132                                                         break;
 133                                         if(i >= autocvar__urllib_nextslot)
 134                                         {
 135                                                 print("url_single_fopen: too many concurrent requests\n");
 136                                                 rdy(NULL, pass, URL_READY_ERROR);
 137                                                 return;
 138                                         }
 139                                 }
 140
 141                                 // GET the data
 142                                 if(!crypto_uri_postbuf(url, i + MIN_URL_ID, string_null, string_null, -1, 0))
 143                                 {
 144                                         print("url_single_fopen: failure in crypto_uri_postbuf\n");
 145                                         rdy(NULL, pass, URL_READY_ERROR);
 146                                         return;
 147                                 }
 148
 149                                 // Make a dummy handle object (no buffers at
 150                                 // all). Wait for data to come from the
 151                                 // server, then call the callback
 152                                 e = spawn();
 153                                 e.classname = "url_single_fopen_file";
 154                                 e.url_url = strzone(url);
 155                                 e.url_fh = URL_FH_CURL;
 156                                 e.url_rbuf = -1;
 157                                 e.url_wbuf = -1;
 158                                 e.url_ready = rdy;
 159                                 e.url_ready_pass = pass;
 160                                 e.url_id = i;
 161                                 url_fromid[i] = e;
 162
 163                                 // make sure this slot won't be reused quickly even on map change
 164                                 cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 165                                 break;
 166                 }
 167         }
 168         else if(url == "-")
 169         {
 170                 switch(mode)
 171                 {
 172                         case FILE_WRITE:
 173                         case FILE_APPEND:
 174                                 e = spawn();
 175                                 e.classname = "url_single_fopen_stdout";
 176                                 e.url_fh = URL_FH_STDOUT;
 177                                 e.url_ready = rdy;
 178                                 e.url_ready_pass = pass;
 179                                 rdy(e, pass, URL_READY_CANWRITE);
 180                                 break;
 181                         case FILE_READ:
 182                                 print("url_single_fopen: cannot open '-' for reading\n");
 183                                 rdy(NULL, pass, URL_READY_ERROR);
 184                                 break;
 185                 }
 186         }
 187         else
 188         {
 189                 float fh;
 190                 fh = fopen(url, mode);
 191                 if(fh < 0)
 192                 {
 193                         rdy(NULL, pass, URL_READY_ERROR);
 194                         return;
 195                 }
 196                 else
 197                 {
 198                         e = spawn();
 199                         e.classname = "url_single_fopen_file";
 200                         e.url_fh = fh;
 201                         e.url_ready = rdy;
 202                         e.url_ready_pass = pass;
 203                         if(mode == FILE_READ)
 204                                 rdy(e, pass, URL_READY_CANREAD);
 205                         else
 206                                 rdy(e, pass, URL_READY_CANWRITE);
 207                 }
 208         }
 209 }
 210
 211 // close a file
 212 void url_fclose(entity e)
 213 {
 214         int i;
 215
 216         if(e.url_fh == URL_FH_CURL)
 217         {
 218                 if(e.url_rbuf == -1 || e.url_wbuf != -1) // not(post GET/POST request)
 219                 if(e.url_rbuf != -1 || e.url_wbuf == -1) // not(pre POST request)
 220                         error("url_fclose: not closable in current state");
 221
 222                 // closing an URL!
 223                 if(e.url_wbuf >= 0)
 224                 {
 225                         // we are closing the write end (HTTP POST request)
 226
 227                         // get slot for HTTP request
 228                         for(i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 229                                 if(url_fromid[i] == NULL)
 230                                         break;
 231                         if(i >= NUM_URL_ID)
 232                         {
 233                                 for(i = 0; i < autocvar__urllib_nextslot; ++i)
 234                                         if(url_fromid[i] == NULL)
 235                                                 break;
 236                                 if(i >= autocvar__urllib_nextslot)
 237                                 {
 238                                         print("url_fclose: too many concurrent requests\n");
 239                                         e.url_ready(e,e.url_ready_pass, URL_READY_ERROR);
 240                                         buf_del(e.url_wbuf);
 241                                         strunzone(e.url_url);
 242                                         remove(e);
 243                                         return;
 244                                 }
 245                         }
 246
 247                         // POST the data
 248                         if(!crypto_uri_postbuf(e.url_url, i + MIN_URL_ID, "text/plain", "", e.url_wbuf, 0))
 249                         {
 250                                 print("url_fclose: failure in crypto_uri_postbuf\n");
 251                                 e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 252                                 buf_del(e.url_wbuf);
 253                                 strunzone(e.url_url);
 254                                 remove(e);
 255                                 return;
 256                         }
 257
 258                         // delete write end. File handle is now in unusable
 259                         // state. Wait for data to come from the server, then
 260                         // call the callback
 261                         buf_del(e.url_wbuf);
 262                         e.url_wbuf = -1;
 263                         e.url_id = i;
 264                         url_fromid[i] = e;
 265
 266                         // make sure this slot won't be reused quickly even on map change
 267                         cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 268                 }
 269                 else
 270                 {
 271                         // we have READ all data, just close
 272                         e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);
 273                         buf_del(e.url_rbuf);
 274                         strunzone(e.url_url);
 275                         remove(e);
 276                 }
 277         }
 278         else if(e.url_fh == URL_FH_STDOUT)
 279         {
 280                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED); // closing creates no reading handle
 281                 remove(e);
 282         }
 283         else
 284         {
 285                 // file
 286                 fclose(e.url_fh);
 287                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED); // closing creates no reading handle
 288                 remove(e);
 289         }
 290 }
 291
 292 // with \n (blame FRIK_FILE)
 293 string url_fgets(entity e)
 294 {
 295         if(e.url_fh == URL_FH_CURL)
 296         {
 297                 if(e.url_rbuf == -1)
 298                         error("url_fgets: not readable in current state");
 299                 // curl
 300                 string s;
 301                 s = bufstr_get(e.url_rbuf, e.url_rbufpos);
 302                 e.url_rbufpos += 1;
 303                 return s;
 304         }
 305         else if(e.url_fh == URL_FH_STDOUT)
 306         {
 307                 // stdout
 308                 return string_null;
 309         }
 310         else
 311         {
 312                 // file
 313                 return fgets(e.url_fh);
 314         }
 315 }
 316
 317 // without \n (blame FRIK_FILE)
 318 void url_fputs(entity e, string s)
 319 {
 320         if(e.url_fh == URL_FH_CURL)
 321         {
 322                 if(e.url_wbuf == -1)
 323                         error("url_fputs: not writable in current state");
 324                 // curl
 325                 bufstr_set(e.url_wbuf, e.url_wbufpos, s);
 326                 e.url_wbufpos += 1;
 327         }
 328         else if(e.url_fh == URL_FH_STDOUT)
 329         {
 330                 // stdout
 331                 print(s);
 332         }
 333         else
 334         {
 335                 // file
 336                 fputs(e.url_fh, s);
 337         }
 338 }
 339
 340 // multi URL object, tries URLs separated by space in sequence
 341 void url_multi_ready(entity fh, entity me, float status)
 342 {
 343         float n;
 344         if(status == URL_READY_ERROR || status < 0)
 345         {
 346                 if(status == -422) // Unprocessable Entity
 347                 {
 348                         print("uri_multi_ready: got HTTP error 422, data is in unusable format - not continuing\n");
 349                         me.url_ready(fh, me.url_ready_pass, status);
 350                         strunzone(me.url_url);
 351                         remove(me);
 352                         return;
 353                 }
 354                 me.url_attempt += 1;
 355                 n = tokenize_console(me.url_url);
 356                 if(n <= me.url_attempt)
 357                 {
 358                         me.url_ready(fh, me.url_ready_pass, status);
 359                         strunzone(me.url_url);
 360                         remove(me);
 361                         return;
 362                 }
 363                 url_single_fopen(argv(me.url_attempt), me.url_mode, url_multi_ready, me);
 364                 return;
 365         }
 366         me.url_ready(fh, me.url_ready_pass, status);
 367 }
 368 void url_multi_fopen(string url, int mode, url_ready_func rdy, entity pass)
 369 {
 370         float n;
 371         n = tokenize_console(url);
 372         if(n <= 0)
 373         {
 374                 print("url_multi_fopen: need at least one URL\n");
 375                 rdy(NULL, pass, URL_READY_ERROR);
 376                 return;
 377         }
 378
 379         entity me;
 380         me = spawn();
 381         me.classname = "url_multi";
 382         me.url_url = strzone(url);
 383         me.url_attempt = 0;
 384         me.url_mode = mode;
 385         me.url_ready = rdy;
 386         me.url_ready_pass = pass;
 387         url_single_fopen(argv(0), mode, url_multi_ready, me);
 388 }