qcsrc/lib/urllib.qc

   1 #include "urllib.qh"
   2
   3 // files
   4 .float url_fh;
   5 const float URL_FH_CURL = -1;
   6 const float URL_FH_STDOUT = -2;
   7
   8 // URLs
   9 .string url_url;
  10 .string url_content_type;
  11 .string url_verb;
  12 .float url_wbuf;
  13 .float url_wbufpos;
  14 .float url_rbuf;
  15 .float url_rbufpos;
  16 .float url_id;
  17 .url_ready_func url_ready;
  18 .entity url_ready_pass;
  19
  20 // for multi handles
  21 .int url_attempt;
  22 .int url_mode;
  23
  24 entity url_fromid[NUM_URL_ID];
  25 int autocvar__urllib_nextslot;
  26
  27 ERASEABLE
  28 float url_URI_Get_Callback(int id, float status, string data)
  29 {
  30         if (id < MIN_URL_ID) return 0;
  31         id -= MIN_URL_ID;
  32         if (id >= NUM_URL_ID) return 0;
  33         entity e;
  34         e = url_fromid[id];
  35         if (!e) return 0;
  36         if (e.url_rbuf >= 0 || e.url_wbuf >= 0)
  37         {
  38                 LOG_INFOF("WARNING: handle %d (%s) has already received data?!?", id + NUM_URL_ID, e.url_url);
  39                 return 0;
  40         }
  41
  42         // whatever happens, we will remove the URL from the list of IDs
  43         url_fromid[id] = NULL;
  44
  45         // if we get here, we MUST have both buffers cleared
  46         if (e.url_rbuf != -1 || e.url_wbuf != -1 || e.url_fh != URL_FH_CURL) error("url_URI_Get_Callback: not a request waiting for data");
  47
  48         if (status == 0)
  49         {
  50                 // WE GOT DATA!
  51                 float n, i;
  52                 n = tokenizebyseparator(data, "\n");
  53                 e.url_rbuf = buf_create();
  54                 if (e.url_rbuf < 0)
  55                 {
  56                         LOG_INFO("url_URI_Get_Callback: out of memory in buf_create");
  57                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  58                         strunzone(e.url_url);
  59                         delete(e);
  60                         return 1;
  61                 }
  62                 e.url_rbufpos = 0;
  63                 if (e.url_rbuf < 0)
  64                 {
  65                         LOG_INFO("url_URI_Get_Callback: out of memory in buf_create");
  66                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
  67                         strunzone(e.url_url);
  68                         delete(e);
  69                         return 1;
  70                 }
  71                 for (i = 0; i < n; ++i)
  72                         bufstr_set(e.url_rbuf, i, argv(i));
  73                 e.url_ready(e, e.url_ready_pass, URL_READY_CANREAD);
  74                 return 1;
  75         }
  76         else
  77         {
  78                 // an ERROR
  79                 e.url_ready(e, e.url_ready_pass, -fabs(status));
  80                 strunzone(e.url_url);
  81                 delete(e);
  82                 return 1;
  83         }
  84 }
  85
  86 ERASEABLE
  87 void url_single_fopen(string url, int mode, url_ready_func rdy, entity pass)
  88 {
  89         entity e;
  90         int i;
  91         if (strstrofs(url, "://", 0) >= 0)
  92         {
  93                 switch (mode)
  94                 {
  95                         case FILE_WRITE:
  96                         case FILE_APPEND:
  97                                 // collect data to a stringbuffer for a POST request
  98                                 // attempts to close will result in a reading handle
  99
 100                                 // create a writing end that does nothing yet
 101                                 e = new_pure(url_single_fopen_file);
 102                                 e.url_url = strzone(url);
 103                                 e.url_content_type = "text/plain";
 104                                 e.url_verb = "";
 105                                 e.url_fh = URL_FH_CURL;
 106                                 e.url_wbuf = buf_create();
 107                                 if (e.url_wbuf < 0)
 108                                 {
 109                                         LOG_INFO("url_single_fopen: out of memory in buf_create");
 110                                         rdy(e, pass, URL_READY_ERROR);
 111                                         strunzone(e.url_url);
 112                                         delete(e);
 113                                         return;
 114                                 }
 115                                 e.url_wbufpos = 0;
 116                                 e.url_rbuf = -1;
 117                                 e.url_ready = rdy;
 118                                 e.url_ready_pass = pass;
 119                                 rdy(e, pass, URL_READY_CANWRITE);
 120                                 break;
 121
 122                         case FILE_READ:
 123                                 // read data only
 124
 125                                 // get slot for HTTP request
 126                                 for (i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 127                                         if (url_fromid[i] == NULL) break;
 128                                 if (i >= NUM_URL_ID)
 129                                 {
 130                                         for (i = 0; i < autocvar__urllib_nextslot; ++i)
 131                                                 if (url_fromid[i] == NULL) break;
 132                                         if (i >= autocvar__urllib_nextslot)
 133                                         {
 134                                                 LOG_INFO("url_single_fopen: too many concurrent requests");
 135                                                 rdy(NULL, pass, URL_READY_ERROR);
 136                                                 return;
 137                                         }
 138                                 }
 139
 140                                 // GET the data
 141                                 if (!crypto_uri_postbuf(url, i + MIN_URL_ID, string_null, string_null, -1, 0))
 142                                 {
 143                                         LOG_INFO("url_single_fopen: failure in crypto_uri_postbuf");
 144                                         rdy(NULL, pass, URL_READY_ERROR);
 145                                         return;
 146                                 }
 147
 148                                 // Make a dummy handle object (no buffers at
 149                                 // all). Wait for data to come from the
 150                                 // server, then call the callback
 151                                 e = new_pure(url_single_fopen_file);
 152                                 e.url_url = strzone(url);
 153                                 e.url_fh = URL_FH_CURL;
 154                                 e.url_rbuf = -1;
 155                                 e.url_wbuf = -1;
 156                                 e.url_ready = rdy;
 157                                 e.url_ready_pass = pass;
 158                                 e.url_id = i;
 159                                 url_fromid[i] = e;
 160
 161                                 // make sure this slot won't be reused quickly even on map change
 162                                 cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 163                                 break;
 164                 }
 165         }
 166         else if (url == "-")
 167         {
 168                 switch (mode)
 169                 {
 170                         case FILE_WRITE:
 171                         case FILE_APPEND:
 172                                 e = new_pure(url_single_fopen_stdout);
 173                                 e.url_fh = URL_FH_STDOUT;
 174                                 e.url_ready = rdy;
 175                                 e.url_ready_pass = pass;
 176                                 rdy(e, pass, URL_READY_CANWRITE);
 177                                 break;
 178                         case FILE_READ:
 179                                 LOG_INFO("url_single_fopen: cannot open '-' for reading");
 180                                 rdy(NULL, pass, URL_READY_ERROR);
 181                                 break;
 182                 }
 183         }
 184         else
 185         {
 186                 float fh;
 187                 fh = fopen(url, mode);
 188                 if (fh < 0)
 189                 {
 190                         rdy(NULL, pass, URL_READY_ERROR);
 191                         return;
 192                 }
 193                 else
 194                 {
 195                         e = new_pure(url_single_fopen_file);
 196                         e.url_fh = fh;
 197                         e.url_ready = rdy;
 198                         e.url_ready_pass = pass;
 199                         if (mode == FILE_READ) rdy(e, pass, URL_READY_CANREAD);
 200                         else rdy(e, pass, URL_READY_CANWRITE);
 201                 }
 202         }
 203 }
 204
 205 // close a file
 206 ERASEABLE
 207 void url_fclose(entity e)
 208 {
 209         int i;
 210
 211         if (e.url_fh == URL_FH_CURL)
 212         {
 213                 if (e.url_rbuf == -1 || e.url_wbuf != -1)     // not(post GET/POST request)
 214                         if (e.url_rbuf != -1 || e.url_wbuf == -1) // not(pre POST request)
 215                                 error("url_fclose: not closable in current state");
 216
 217                 // closing an URL!
 218                 if (e.url_wbuf >= 0)
 219                 {
 220                         // we are closing the write end (HTTP POST request)
 221
 222                         // get slot for HTTP request
 223                         for (i = autocvar__urllib_nextslot; i < NUM_URL_ID; ++i)
 224                                 if (url_fromid[i] == NULL) break;
 225                         if (i >= NUM_URL_ID)
 226                         {
 227                                 for (i = 0; i < autocvar__urllib_nextslot; ++i)
 228                                         if (url_fromid[i] == NULL) break;
 229                                 if (i >= autocvar__urllib_nextslot)
 230                                 {
 231                                         LOG_INFO("url_fclose: too many concurrent requests");
 232                                         e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 233                                         buf_del(e.url_wbuf);
 234                                         strunzone(e.url_url);
 235                                         delete(e);
 236                                         return;
 237                                 }
 238                         }
 239
 240                         // POST the data
 241                         if (!crypto_uri_postbuf(e.url_url, i + MIN_URL_ID, e.url_content_type, e.url_verb, e.url_wbuf, 0))
 242                         {
 243                                 LOG_INFO("url_fclose: failure in crypto_uri_postbuf");
 244                                 e.url_ready(e, e.url_ready_pass, URL_READY_ERROR);
 245                                 buf_del(e.url_wbuf);
 246                                 strunzone(e.url_url);
 247                                 delete(e);
 248                                 return;
 249                         }
 250
 251                         // delete write end. File handle is now in unusable
 252                         // state. Wait for data to come from the server, then
 253                         // call the callback
 254                         buf_del(e.url_wbuf);
 255                         e.url_wbuf = -1;
 256                         e.url_id = i;
 257                         url_fromid[i] = e;
 258
 259                         // make sure this slot won't be reused quickly even on map change
 260                         cvar_set("_urllib_nextslot", ftos((i + 1) % NUM_URL_ID));
 261                 }
 262                 else
 263                 {
 264                         // we have READ all data, just close
 265                         e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);
 266                         buf_del(e.url_rbuf);
 267                         strunzone(e.url_url);
 268                         delete(e);
 269                 }
 270         }
 271         else if (e.url_fh == URL_FH_STDOUT)
 272         {
 273                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);  // closing creates no reading handle
 274                 delete(e);
 275         }
 276         else
 277         {
 278                 // file
 279                 fclose(e.url_fh);
 280                 e.url_ready(e, e.url_ready_pass, URL_READY_CLOSED);  // closing creates no reading handle
 281                 delete(e);
 282         }
 283 }
 284
 285 // with \n (blame FRIK_FILE)
 286 ERASEABLE
 287 string url_fgets(entity e)
 288 {
 289         if (e.url_fh == URL_FH_CURL)
 290         {
 291                 if (e.url_rbuf == -1) error("url_fgets: not readable in current state");
 292                 // curl
 293                 string s;
 294                 s = bufstr_get(e.url_rbuf, e.url_rbufpos);
 295                 e.url_rbufpos += 1;
 296                 return s;
 297         }
 298         else if (e.url_fh == URL_FH_STDOUT)
 299         {
 300                 // stdout
 301                 return string_null;
 302         }
 303         else
 304         {
 305                 // file
 306                 return fgets(e.url_fh);
 307         }
 308 }
 309
 310 // without \n (blame FRIK_FILE)
 311 ERASEABLE
 312 void url_fputs(entity e, string s)
 313 {
 314         if (e.url_fh == URL_FH_CURL)
 315         {
 316                 if (e.url_wbuf == -1) error("url_fputs: not writable in current state");
 317                 // curl
 318                 bufstr_set(e.url_wbuf, e.url_wbufpos, s);
 319                 e.url_wbufpos += 1;
 320         }
 321         else if (e.url_fh == URL_FH_STDOUT)
 322         {
 323                 // stdout
 324                 print(s);
 325         }
 326         else
 327         {
 328                 // file
 329                 fputs(e.url_fh, s);
 330         }
 331 }
 332
 333 // multi URL object, tries URLs separated by space in sequence
 334 ERASEABLE
 335 void url_multi_ready(entity fh, entity me, float status)
 336 {
 337         float n;
 338         if (status == URL_READY_ERROR || status < 0)
 339         {
 340                 if (status == -422)  // Unprocessable Entity
 341                 {
 342                         LOG_INFO("uri_multi_ready: got HTTP error 422, data is in unusable format - not continuing");
 343                         me.url_ready(fh, me.url_ready_pass, status);
 344                         strunzone(me.url_url);
 345                         delete(me);
 346                         return;
 347                 }
 348                 me.url_attempt += 1;
 349                 n = tokenize_console(me.url_url);
 350                 if (n <= me.url_attempt)
 351                 {
 352                         me.url_ready(fh, me.url_ready_pass, status);
 353                         strunzone(me.url_url);
 354                         delete(me);
 355                         return;
 356                 }
 357                 url_single_fopen(argv(me.url_attempt), me.url_mode, url_multi_ready, me);
 358                 return;
 359         }
 360         me.url_ready(fh, me.url_ready_pass, status);
 361 }
 362
 363 ERASEABLE
 364 void url_multi_fopen(string url, int mode, url_ready_func rdy, entity pass)
 365 {
 366         float n;
 367         n = tokenize_console(url);
 368         if (n <= 0)
 369         {
 370                 LOG_INFO("url_multi_fopen: need at least one URL");
 371                 rdy(NULL, pass, URL_READY_ERROR);
 372                 return;
 373         }
 374
 375         entity me = new_pure(url_multi);
 376         me.url_url = strzone(url);
 377         me.url_attempt = 0;
 378         me.url_mode = mode;
 379         me.url_ready = rdy;
 380         me.url_ready_pass = pass;
 381         url_single_fopen(argv(0), mode, url_multi_ready, me);
 382 }