qcsrc/lib/urllib.qc

   1 #include "urllib.qh"
   2
   3 // files
   4 .float url_fh;
   5 const float URL_FH_CURL = -1;
   6 const float URL_FH_STDOUT = -2;
   7
   8 // URLs
   9 .string url_url;
  10 .float url_wbuf;
  11 .float url_wbufpos;
  12 .float url_rbuf;
  13 .float url_rbufpos;
  14 .float url_id;
  15 .url_ready_func url_ready;
  16 .entity url_ready_pass;
  17
  18 // for multi handles
  19 .int url_attempt;
  20 .int url_mode;
  21
  22 entity url_fromid[NUM_URL_ID];
  23 int autocvar__urllib_nextslot;
  24
  25 float url_URI_Get_Callback(int id, float status, string data)
  26 {
  27         if (id < MIN_URL_ID) return 0;
  28         id -= MIN_URL_ID;
  29         if (id >= NUM_URL_ID) return 0;
  30         entity e;
  31         e = url_fromid[id];
  32         if (!e) return 0;
  33         if (e.url_rbuf >= 0 || e.url_wbuf >= 0)
  34         {
  35                 LOG_INFOF("WARNING: handle %d (%s) has already received data?!?\n", id + NUM_URL_ID, e.url_url);
  36                 return 0;
  37         }
  38
  39         // whatever happens, we will remove the URL from the list of IDs
  40         url_fromid[id] = NULL;
  41
  42         // if we get here, we MUST have both buffers cleared
  43         if (e.url_rbuf != -1 || e.url_wbuf != -1 || e.url_fh != URL_FH_CURL) error("url_URI_Get_Callback: not a request waiting for data");
  44
  45         if (status == 0)
  46         {
  47                 // WE GOT DATA!
  48                 float n, i;
  49                 n = tokenizebyseparator(data, "\n");
  50                 e.url_rbuf = buf_create();
  51                 if (e.url_rbuf < 0)
  52                 {
  53                         LOG_INFO("url_URI_Get_Callback: out of memory in buf_create\n");
  54                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  55                         strunzone(e.url_url);
  56                         remove(e);
  57                         return 1;
  58                 }
  59                 e.url_rbufpos = 0;
  60                 if (e.url_rbuf < 0)
  61                 {
  62                         LOG_INFO("url_URI_Get_Callback: out of memory in buf_create\n");
  63                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  64                         strunzone(e.url_url);
  65                         remove(e);
  66                         return 1;
  67                 }
  68                 for (i = 0; i < n; ++i)
  69                         bufstr_set(e.url_rbuf, i, argv(i));
  70                 e.url_ready(e, e.url_ready_pass, URL_READY_CANREAD);
  71                 return 1;
  72         }
  73         else
  74         {
  75                 // an ERROR
  76                 e.url_ready(e, e.url_ready_pass, -fabs(status));
  77                 strunzone(e.url_url);
  78                 remove(e);
  79                 return 1;
  80         }
  81 }
  82
  83 void url_single_fopen(string url, int mode, url_ready_func rdy, entity pass)
  84 {
  85         entity e;
  86         int i;
  87         if (strstrofs(url, "://", 0) >= 0)
  88         {
  89                 switch (mode)
  90                 {
  91                         case FILE_WRITE:
  92                         case FILE_APPEND:
  93                                 // collect data to a stringbuffer for a POST request
  94                                 // attempts to close will result in a reading handle
  95
  96                                 // create a writing end that does nothing yet
  97                                 e = new_pure(url_single_fopen_file);
  98                                 e.url_url = strzone(url);
  99                                 e.url_fh = URL_FH_CURL;
 100                                 e.url_wbuf = buf_create();
 101                                 if (e.url_wbuf < 0)
 102                                 {
 103                                         LOG_INFO("url_single_fopen: out of memory in buf_create\n");
 104                                         rdy(e, pass, URL_READY_ERROR);
 105                                         strunzone(e.url_url);
 106                                         remove(e);
 107                                         return;
 108                                 }
 109                                 e.url_wbufpos = 0;
 110                                 e.url_rbuf = -1;
 111                                 e.url_ready = rdy;
 112                                 e.url_ready_pass = pass;
 113                                 rdy(e, pass, URL_READY_CANWRITE);
 114                                 break;
 115
 116                         case FILE_READ:
 117                                 // read data only
 118
 119                                 // get slot for HTTP request
 120                                 for (i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 121                                         if (url_fromid[i] == NULL) break;
 122                                 if (i >= NUM_URL_ID)
 123                                 {
 124                                         for (i = 0; i < autocvar__urllib_nextslot; ++i)
 125                                                 if (url_fromid[i] == NULL) break;
 126                                         if (i >= autocvar__urllib_nextslot)
 127                                         {
 128                                                 LOG_INFO("url_single_fopen: too many concurrent requests\n");
 129                                                 rdy(NULL, pass, URL_READY_ERROR);
 130                                                 return;
 131                                         }
 132                                 }
 133
 134                                 // GET the data
 135                                 if (!crypto_uri_postbuf(url, i + MIN_URL_ID, string_null, string_null, -1, 0))
 136                                 {
 137                                         LOG_INFO("url_single_fopen: failure in crypto_uri_postbuf\n");
 138                                         rdy(NULL, pass, URL_READY_ERROR);
 139                                         return;
 140                                 }
 141
 142                                 // Make a dummy handle object (no buffers at
 143                                 // all). Wait for data to come from the
 144                                 // server, then call the callback
 145                                 e = new_pure(url_single_fopen_file);
 146                                 e.url_url = strzone(url);
 147                                 e.url_fh = URL_FH_CURL;
 148                                 e.url_rbuf = -1;
 149                                 e.url_wbuf = -1;
 150                                 e.url_ready = rdy;
 151                                 e.url_ready_pass = pass;
 152                                 e.url_id = i;
 153                                 url_fromid[i] = e;
 154
 155                                 // make sure this slot won't be reused quickly even on map change
 156                                 cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 157                                 break;
 158                 }
 159         }
 160         else if (url == "-")
 161         {
 162                 switch (mode)
 163                 {
 164                         case FILE_WRITE:
 165                         case FILE_APPEND:
 166                                 e = new_pure(url_single_fopen_stdout);
 167                                 e.url_fh = URL_FH_STDOUT;
 168                                 e.url_ready = rdy;
 169                                 e.url_ready_pass = pass;
 170                                 rdy(e, pass, URL_READY_CANWRITE);
 171                                 break;
 172                         case FILE_READ:
 173                                 LOG_INFO("url_single_fopen: cannot open '-' for reading\n");
 174                                 rdy(NULL, pass, URL_READY_ERROR);
 175                                 break;
 176                 }
 177         }
 178         else
 179         {
 180                 float fh;
 181                 fh = fopen(url, mode);
 182                 if (fh < 0)
 183                 {
 184                         rdy(NULL, pass, URL_READY_ERROR);
 185                         return;
 186                 }
 187                 else
 188                 {
 189                         e = new_pure(url_single_fopen_file);
 190                         e.url_fh = fh;
 191                         e.url_ready = rdy;
 192                         e.url_ready_pass = pass;
 193                         if (mode == FILE_READ) rdy(e, pass, URL_READY_CANREAD);
 194                         else rdy(e, pass, URL_READY_CANWRITE);
 195                 }
 196         }
 197 }
 198
 199 // close a file
 200 void url_fclose(entity e)
 201 {
 202         int i;
 203
 204         if (e.url_fh == URL_FH_CURL)
 205         {
 206                 if (e.url_rbuf == -1 || e.url_wbuf != -1)     // not(post GET/POST request)
 207                         if (e.url_rbuf != -1 || e.url_wbuf == -1) // not(pre POST request)
 208                                 error("url_fclose: not closable in current state");
 209
 210                 // closing an URL!
 211                 if (e.url_wbuf >= 0)
 212                 {
 213                         // we are closing the write end (HTTP POST request)
 214
 215                         // get slot for HTTP request
 216                         for (i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 217                                 if (url_fromid[i] == NULL) break;
 218                         if (i >= NUM_URL_ID)
 219                         {
 220                                 for (i = 0; i < autocvar__urllib_nextslot; ++i)
 221                                         if (url_fromid[i] == NULL) break;
 222                                 if (i >= autocvar__urllib_nextslot)
 223                                 {
 224                                         LOG_INFO("url_fclose: too many concurrent requests\n");
 225                                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 226                                         buf_del(e.url_wbuf);
 227                                         strunzone(e.url_url);
 228                                         remove(e);
 229                                         return;
 230                                 }
 231                         }
 232
 233                         // POST the data
 234                         if (!crypto_uri_postbuf(e.url_url, i + MIN_URL_ID, "text/plain", "", e.url_wbuf, 0))
 235                         {
 236                                 LOG_INFO("url_fclose: failure in crypto_uri_postbuf\n");
 237                                 e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 238                                 buf_del(e.url_wbuf);
 239                                 strunzone(e.url_url);
 240                                 remove(e);
 241                                 return;
 242                         }
 243
 244                         // delete write end. File handle is now in unusable
 245                         // state. Wait for data to come from the server, then
 246                         // call the callback
 247                         buf_del(e.url_wbuf);
 248                         e.url_wbuf = -1;
 249                         e.url_id = i;
 250                         url_fromid[i] = e;
 251
 252                         // make sure this slot won't be reused quickly even on map change
 253                         cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 254                 }
 255                 else
 256                 {
 257                         // we have READ all data, just close
 258                         e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);
 259                         buf_del(e.url_rbuf);
 260                         strunzone(e.url_url);
 261                         remove(e);
 262                 }
 263         }
 264         else if (e.url_fh == URL_FH_STDOUT)
 265         {
 266                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);  // closing creates no reading handle
 267                 remove(e);
 268         }
 269         else
 270         {
 271                 // file
 272                 fclose(e.url_fh);
 273                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);  // closing creates no reading handle
 274                 remove(e);
 275         }
 276 }
 277
 278 // with \n (blame FRIK_FILE)
 279 string url_fgets(entity e)
 280 {
 281         if (e.url_fh == URL_FH_CURL)
 282         {
 283                 if (e.url_rbuf == -1) error("url_fgets: not readable in current state");
 284                 // curl
 285                 string s;
 286                 s = bufstr_get(e.url_rbuf, e.url_rbufpos);
 287                 e.url_rbufpos += 1;
 288                 return s;
 289         }
 290         else if (e.url_fh == URL_FH_STDOUT)
 291         {
 292                 // stdout
 293                 return string_null;
 294         }
 295         else
 296         {
 297                 // file
 298                 return fgets(e.url_fh);
 299         }
 300 }
 301
 302 // without \n (blame FRIK_FILE)
 303 void url_fputs(entity e, string s)
 304 {
 305         if (e.url_fh == URL_FH_CURL)
 306         {
 307                 if (e.url_wbuf == -1) error("url_fputs: not writable in current state");
 308                 // curl
 309                 bufstr_set(e.url_wbuf, e.url_wbufpos, s);
 310                 e.url_wbufpos += 1;
 311         }
 312         else if (e.url_fh == URL_FH_STDOUT)
 313         {
 314                 // stdout
 315                 LOG_INFO(s);
 316         }
 317         else
 318         {
 319                 // file
 320                 fputs(e.url_fh, s);
 321         }
 322 }
 323
 324 // multi URL object, tries URLs separated by space in sequence
 325 void url_multi_ready(entity fh, entity me, float status)
 326 {
 327         float n;
 328         if (status == URL_READY_ERROR || status < 0)
 329         {
 330                 if (status == -422)  // Unprocessable Entity
 331                 {
 332                         LOG_INFO("uri_multi_ready: got HTTP error 422, data is in unusable format - not continuing\n");
 333                         me.url_ready(fh, me.url_ready_pass, status);
 334                         strunzone(me.url_url);
 335                         remove(me);
 336                         return;
 337                 }
 338                 me.url_attempt += 1;
 339                 n = tokenize_console(me.url_url);
 340                 if (n <= me.url_attempt)
 341                 {
 342                         me.url_ready(fh, me.url_ready_pass, status);
 343                         strunzone(me.url_url);
 344                         remove(me);
 345                         return;
 346                 }
 347                 url_single_fopen(argv(me.url_attempt), me.url_mode, url_multi_ready, me);
 348                 return;
 349         }
 350         me.url_ready(fh, me.url_ready_pass, status);
 351 }
 352 void url_multi_fopen(string url, int mode, url_ready_func rdy, entity pass)
 353 {
 354         float n;
 355         n = tokenize_console(url);
 356         if (n <= 0)
 357         {
 358                 LOG_INFO("url_multi_fopen: need at least one URL\n");
 359                 rdy(NULL, pass, URL_READY_ERROR);
 360                 return;
 361         }
 362
 363         entity me = new_pure(url_multi);
 364         me.url_url = strzone(url);
 365         me.url_attempt = 0;
 366         me.url_mode = mode;
 367         me.url_ready = rdy;
 368         me.url_ready_pass = pass;
 369         url_single_fopen(argv(0), mode, url_multi_ready, me);
 370 }