qcsrc/lib/urllib.qc

   1 #include "urllib.qh"
   2
   3 // files
   4 .float url_fh;
   5 const float URL_FH_CURL = -1;
   6 const float URL_FH_STDOUT = -2;
   7
   8 // URLs
   9 .string url_url;
  10 .string url_content_type;
  11 .string url_verb;
  12 .float url_wbuf;
  13 .float url_wbufpos;
  14 .float url_rbuf;
  15 .float url_rbufpos;
  16 .float url_id;
  17 .url_ready_func url_ready;
  18 .entity url_ready_pass;
  19
  20 // for multi handles
  21 .int url_attempt;
  22 .int url_mode;
  23
  24 entity url_fromid[NUM_URL_ID];
  25 int autocvar__urllib_nextslot;
  26
  27 float url_URI_Get_Callback(int id, float status, string data)
  28 {
  29         if (id < MIN_URL_ID) return 0;
  30         id -= MIN_URL_ID;
  31         if (id >= NUM_URL_ID) return 0;
  32         entity e;
  33         e = url_fromid[id];
  34         if (!e) return 0;
  35         if (e.url_rbuf >= 0 || e.url_wbuf >= 0)
  36         {
  37                 LOG_INFOF("WARNING: handle %d (%s) has already received data?!?\n", id + NUM_URL_ID, e.url_url);
  38                 return 0;
  39         }
  40
  41         // whatever happens, we will remove the URL from the list of IDs
  42         url_fromid[id] = NULL;
  43
  44         // if we get here, we MUST have both buffers cleared
  45         if (e.url_rbuf != -1 || e.url_wbuf != -1 || e.url_fh != URL_FH_CURL) error("url_URI_Get_Callback: not a request waiting for data");
  46
  47         if (status == 0)
  48         {
  49                 // WE GOT DATA!
  50                 float n, i;
  51                 n = tokenizebyseparator(data, "\n");
  52                 e.url_rbuf = buf_create();
  53                 if (e.url_rbuf < 0)
  54                 {
  55                         LOG_INFO("url_URI_Get_Callback: out of memory in buf_create\n");
  56                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  57                         strunzone(e.url_url);
  58                         delete(e);
  59                         return 1;
  60                 }
  61                 e.url_rbufpos = 0;
  62                 if (e.url_rbuf < 0)
  63                 {
  64                         LOG_INFO("url_URI_Get_Callback: out of memory in buf_create\n");
  65                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  66                         strunzone(e.url_url);
  67                         delete(e);
  68                         return 1;
  69                 }
  70                 for (i = 0; i < n; ++i)
  71                         bufstr_set(e.url_rbuf, i, argv(i));
  72                 e.url_ready(e, e.url_ready_pass, URL_READY_CANREAD);
  73                 return 1;
  74         }
  75         else
  76         {
  77                 // an ERROR
  78                 e.url_ready(e, e.url_ready_pass, -fabs(status));
  79                 strunzone(e.url_url);
  80                 delete(e);
  81                 return 1;
  82         }
  83 }
  84
  85 void url_single_fopen(string url, int mode, url_ready_func rdy, entity pass)
  86 {
  87         entity e;
  88         int i;
  89         if (strstrofs(url, "://", 0) >= 0)
  90         {
  91                 switch (mode)
  92                 {
  93                         case FILE_WRITE:
  94                         case FILE_APPEND:
  95                                 // collect data to a stringbuffer for a POST request
  96                                 // attempts to close will result in a reading handle
  97
  98                                 // create a writing end that does nothing yet
  99                                 e = new_pure(url_single_fopen_file);
 100                                 e.url_url = strzone(url);
 101                                 e.url_content_type = "text/plain";
 102                                 e.url_verb = "";
 103                                 e.url_fh = URL_FH_CURL;
 104                                 e.url_wbuf = buf_create();
 105                                 if (e.url_wbuf < 0)
 106                                 {
 107                                         LOG_INFO("url_single_fopen: out of memory in buf_create\n");
 108                                         rdy(e, pass, URL_READY_ERROR);
 109                                         strunzone(e.url_url);
 110                                         delete(e);
 111                                         return;
 112                                 }
 113                                 e.url_wbufpos = 0;
 114                                 e.url_rbuf = -1;
 115                                 e.url_ready = rdy;
 116                                 e.url_ready_pass = pass;
 117                                 rdy(e, pass, URL_READY_CANWRITE);
 118                                 break;
 119
 120                         case FILE_READ:
 121                                 // read data only
 122
 123                                 // get slot for HTTP request
 124                                 for (i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 125                                         if (url_fromid[i] == NULL) break;
 126                                 if (i >= NUM_URL_ID)
 127                                 {
 128                                         for (i = 0; i < autocvar__urllib_nextslot; ++i)
 129                                                 if (url_fromid[i] == NULL) break;
 130                                         if (i >= autocvar__urllib_nextslot)
 131                                         {
 132                                                 LOG_INFO("url_single_fopen: too many concurrent requests\n");
 133                                                 rdy(NULL, pass, URL_READY_ERROR);
 134                                                 return;
 135                                         }
 136                                 }
 137
 138                                 // GET the data
 139                                 if (!crypto_uri_postbuf(url, i + MIN_URL_ID, string_null, string_null, -1, 0))
 140                                 {
 141                                         LOG_INFO("url_single_fopen: failure in crypto_uri_postbuf\n");
 142                                         rdy(NULL, pass, URL_READY_ERROR);
 143                                         return;
 144                                 }
 145
 146                                 // Make a dummy handle object (no buffers at
 147                                 // all). Wait for data to come from the
 148                                 // server, then call the callback
 149                                 e = new_pure(url_single_fopen_file);
 150                                 e.url_url = strzone(url);
 151                                 e.url_fh = URL_FH_CURL;
 152                                 e.url_rbuf = -1;
 153                                 e.url_wbuf = -1;
 154                                 e.url_ready = rdy;
 155                                 e.url_ready_pass = pass;
 156                                 e.url_id = i;
 157                                 url_fromid[i] = e;
 158
 159                                 // make sure this slot won't be reused quickly even on map change
 160                                 cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 161                                 break;
 162                 }
 163         }
 164         else if (url == "-")
 165         {
 166                 switch (mode)
 167                 {
 168                         case FILE_WRITE:
 169                         case FILE_APPEND:
 170                                 e = new_pure(url_single_fopen_stdout);
 171                                 e.url_fh = URL_FH_STDOUT;
 172                                 e.url_ready = rdy;
 173                                 e.url_ready_pass = pass;
 174                                 rdy(e, pass, URL_READY_CANWRITE);
 175                                 break;
 176                         case FILE_READ:
 177                                 LOG_INFO("url_single_fopen: cannot open '-' for reading\n");
 178                                 rdy(NULL, pass, URL_READY_ERROR);
 179                                 break;
 180                 }
 181         }
 182         else
 183         {
 184                 float fh;
 185                 fh = fopen(url, mode);
 186                 if (fh < 0)
 187                 {
 188                         rdy(NULL, pass, URL_READY_ERROR);
 189                         return;
 190                 }
 191                 else
 192                 {
 193                         e = new_pure(url_single_fopen_file);
 194                         e.url_fh = fh;
 195                         e.url_ready = rdy;
 196                         e.url_ready_pass = pass;
 197                         if (mode == FILE_READ) rdy(e, pass, URL_READY_CANREAD);
 198                         else rdy(e, pass, URL_READY_CANWRITE);
 199                 }
 200         }
 201 }
 202
 203 // close a file
 204 void url_fclose(entity e)
 205 {
 206         int i;
 207
 208         if (e.url_fh == URL_FH_CURL)
 209         {
 210                 if (e.url_rbuf == -1 || e.url_wbuf != -1)     // not(post GET/POST request)
 211                         if (e.url_rbuf != -1 || e.url_wbuf == -1) // not(pre POST request)
 212                                 error("url_fclose: not closable in current state");
 213
 214                 // closing an URL!
 215                 if (e.url_wbuf >= 0)
 216                 {
 217                         // we are closing the write end (HTTP POST request)
 218
 219                         // get slot for HTTP request
 220                         for (i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 221                                 if (url_fromid[i] == NULL) break;
 222                         if (i >= NUM_URL_ID)
 223                         {
 224                                 for (i = 0; i < autocvar__urllib_nextslot; ++i)
 225                                         if (url_fromid[i] == NULL) break;
 226                                 if (i >= autocvar__urllib_nextslot)
 227                                 {
 228                                         LOG_INFO("url_fclose: too many concurrent requests\n");
 229                                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 230                                         buf_del(e.url_wbuf);
 231                                         strunzone(e.url_url);
 232                                         delete(e);
 233                                         return;
 234                                 }
 235                         }
 236
 237                         // POST the data
 238                         if (!crypto_uri_postbuf(e.url_url, i + MIN_URL_ID, e.url_content_type, e.url_verb, e.url_wbuf, 0))
 239                         {
 240                                 LOG_INFO("url_fclose: failure in crypto_uri_postbuf\n");
 241                                 e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 242                                 buf_del(e.url_wbuf);
 243                                 strunzone(e.url_url);
 244                                 delete(e);
 245                                 return;
 246                         }
 247
 248                         // delete write end. File handle is now in unusable
 249                         // state. Wait for data to come from the server, then
 250                         // call the callback
 251                         buf_del(e.url_wbuf);
 252                         e.url_wbuf = -1;
 253                         e.url_id = i;
 254                         url_fromid[i] = e;
 255
 256                         // make sure this slot won't be reused quickly even on map change
 257                         cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 258                 }
 259                 else
 260                 {
 261                         // we have READ all data, just close
 262                         e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);
 263                         buf_del(e.url_rbuf);
 264                         strunzone(e.url_url);
 265                         delete(e);
 266                 }
 267         }
 268         else if (e.url_fh == URL_FH_STDOUT)
 269         {
 270                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);  // closing creates no reading handle
 271                 delete(e);
 272         }
 273         else
 274         {
 275                 // file
 276                 fclose(e.url_fh);
 277                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);  // closing creates no reading handle
 278                 delete(e);
 279         }
 280 }
 281
 282 // with \n (blame FRIK_FILE)
 283 string url_fgets(entity e)
 284 {
 285         if (e.url_fh == URL_FH_CURL)
 286         {
 287                 if (e.url_rbuf == -1) error("url_fgets: not readable in current state");
 288                 // curl
 289                 string s;
 290                 s = bufstr_get(e.url_rbuf, e.url_rbufpos);
 291                 e.url_rbufpos += 1;
 292                 return s;
 293         }
 294         else if (e.url_fh == URL_FH_STDOUT)
 295         {
 296                 // stdout
 297                 return string_null;
 298         }
 299         else
 300         {
 301                 // file
 302                 return fgets(e.url_fh);
 303         }
 304 }
 305
 306 // without \n (blame FRIK_FILE)
 307 void url_fputs(entity e, string s)
 308 {
 309         if (e.url_fh == URL_FH_CURL)
 310         {
 311                 if (e.url_wbuf == -1) error("url_fputs: not writable in current state");
 312                 // curl
 313                 bufstr_set(e.url_wbuf, e.url_wbufpos, s);
 314                 e.url_wbufpos += 1;
 315         }
 316         else if (e.url_fh == URL_FH_STDOUT)
 317         {
 318                 // stdout
 319                 print(s);
 320         }
 321         else
 322         {
 323                 // file
 324                 fputs(e.url_fh, s);
 325         }
 326 }
 327
 328 // multi URL object, tries URLs separated by space in sequence
 329 void url_multi_ready(entity fh, entity me, float status)
 330 {
 331         float n;
 332         if (status == URL_READY_ERROR || status < 0)
 333         {
 334                 if (status == -422)  // Unprocessable Entity
 335                 {
 336                         LOG_INFO("uri_multi_ready: got HTTP error 422, data is in unusable format - not continuing\n");
 337                         me.url_ready(fh, me.url_ready_pass, status);
 338                         strunzone(me.url_url);
 339                         delete(me);
 340                         return;
 341                 }
 342                 me.url_attempt += 1;
 343                 n = tokenize_console(me.url_url);
 344                 if (n <= me.url_attempt)
 345                 {
 346                         me.url_ready(fh, me.url_ready_pass, status);
 347                         strunzone(me.url_url);
 348                         delete(me);
 349                         return;
 350                 }
 351                 url_single_fopen(argv(me.url_attempt), me.url_mode, url_multi_ready, me);
 352                 return;
 353         }
 354         me.url_ready(fh, me.url_ready_pass, status);
 355 }
 356 void url_multi_fopen(string url, int mode, url_ready_func rdy, entity pass)
 357 {
 358         float n;
 359         n = tokenize_console(url);
 360         if (n <= 0)
 361         {
 362                 LOG_INFO("url_multi_fopen: need at least one URL\n");
 363                 rdy(NULL, pass, URL_READY_ERROR);
 364                 return;
 365         }
 366
 367         entity me = new_pure(url_multi);
 368         me.url_url = strzone(url);
 369         me.url_attempt = 0;
 370         me.url_mode = mode;
 371         me.url_ready = rdy;
 372         me.url_ready_pass = pass;
 373         url_single_fopen(argv(0), mode, url_multi_ready, me);
 374 }