Support teamscores when parsing, but do not store them yet.

[xonotic/xonstat.git] / xonstat / views / submission.py

diff --git a/xonstat/views/submission.py b/xonstat/views/submission.py
old mode 100755 (executable)
new mode 100644 (file)
index ea3e5c1..b002454
--- a/xonstat/views/submission.py
+++ b/xonstat/views/submission.py
@@ -1,28 +1,182 @@
 import datetime\r
 import logging\r
 import datetime\r
 import logging\r

+import os\r

 import pyramid.httpexceptions\r
 import re\r
 import time\r
 import pyramid.httpexceptions\r
 import re\r
 import time\r

-from pyramid.config import get_current_registry\r
+import sqlalchemy.sql.expression as expr\r

 from pyramid.response import Response\r
 from pyramid.response import Response\r

+from sqlalchemy import Sequence\r

 from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound\r
 from xonstat.d0_blind_id import d0_blind_id_verify\r
 from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound\r
 from xonstat.d0_blind_id import d0_blind_id_verify\r

+from xonstat.elo import process_elos\r

 from xonstat.models import *\r
 from xonstat.models import *\r

-from xonstat.util import strip_colors\r
+from xonstat.util import strip_colors, qfont_decode\r
+\r

 \r
 log = logging.getLogger(__name__)\r
 \r
 \r
 log = logging.getLogger(__name__)\r
 \r

-def is_supported_gametype(gametype):\r
+\r
+def parse_stats_submission(body):\r
+    """\r
+    Parses the POST request body for a stats submission\r
+    """\r
+    # storage vars for the request body\r
+    game_meta = {}\r
+    events = {}\r
+    players = []\r
+    teams = []\r
+\r
+    # we're not in either stanza to start\r
+    in_P = in_Q = False\r
+\r
+    for line in body.split('\n'):\r
+        try:\r
+            (key, value) = line.strip().split(' ', 1)\r
+\r
+            # Server (S) and Nick (n) fields can have international characters.\r
+            if key in 'S' 'n':\r
+                value = unicode(value, 'utf-8')\r
+\r
+            if key not in 'P' 'Q' 'n' 'e' 't' 'i':\r
+                game_meta[key] = value\r
+\r
+            if key == 'Q' or key == 'P':\r
+                #log.debug('Found a {0}'.format(key))\r
+                #log.debug('in_Q: {0}'.format(in_Q))\r
+                #log.debug('in_P: {0}'.format(in_P))\r
+                #log.debug('events: {0}'.format(events))\r
+\r
+                # check where we were before and append events accordingly\r
+                if in_Q and len(events) > 0:\r
+                    #log.debug('creating a team (Q) entry')\r
+                    teams.append(events)\r
+                    events = {}\r
+                elif in_P and len(events) > 0:\r
+                    #log.debug('creating a player (P) entry')\r
+                    players.append(events)\r
+                    events = {}\r
+\r
+                if key == 'P':\r
+                    #log.debug('key == P')\r
+                    in_P = True\r
+                    in_Q = False\r
+                elif key == 'Q':\r
+                    #log.debug('key == Q')\r
+                    in_P = False\r
+                    in_Q = True\r
+\r
+                events[key] = value\r
+\r
+            if key == 'e':\r
+                (subkey, subvalue) = value.split(' ', 1)\r
+                events[subkey] = subvalue\r
+            if key == 'n':\r
+                events[key] = value\r
+            if key == 't':\r
+                events[key] = value\r
+        except:\r
+            # no key/value pair - move on to the next line\r
+            pass\r
+\r
+    # add the last entity we were working on\r
+    if in_P and len(events) > 0:\r
+        players.append(events)\r
+    elif in_Q and len(events) > 0:\r
+        teams.append(events)\r
+\r
+    return (game_meta, players, teams)\r
+\r
+\r
+def is_blank_game(gametype, players):\r
+    """Determine if this is a blank game or not. A blank game is either:\r
+\r
+    1) a match that ended in the warmup stage, where accuracy events are not\r
+    present (for non-CTS games)\r
+\r
+    2) a match in which no player made a positive or negative score AND was\r
+    on the scoreboard\r
+\r
+    ... or for CTS, which doesn't record accuracy events\r
+\r
+    1) a match in which no player made a fastest lap AND was\r
+    on the scoreboard\r
+    """\r
+    r = re.compile(r'acc-.*-cnt-fired')\r
+    flg_nonzero_score = False\r
+    flg_acc_events = False\r
+    flg_fastest_lap = False\r
+\r
+    for events in players:\r
+        if is_real_player(events) and played_in_game(events):\r
+            for (key,value) in events.items():\r
+                if key == 'scoreboard-score' and value != 0:\r
+                    flg_nonzero_score = True\r
+                if r.search(key):\r
+                    flg_acc_events = True\r
+                if key == 'scoreboard-fastest':\r
+                    flg_fastest_lap = True\r
+\r
+    if gametype == 'cts':\r
+        return not flg_fastest_lap\r
+    else:\r
+        return not (flg_nonzero_score and flg_acc_events)\r
+\r
+\r
+def get_remote_addr(request):\r
+    """Get the Xonotic server's IP address"""\r
+    if 'X-Forwarded-For' in request.headers:\r
+        return request.headers['X-Forwarded-For']\r
+    else:\r
+        return request.remote_addr\r
+\r
+\r
+def is_supported_gametype(gametype, version):\r

     """Whether a gametype is supported or not"""\r
     """Whether a gametype is supported or not"""\r

-    flg_supported = True\r
+    is_supported = False\r
+\r
+    # if the type can be supported, but with version constraints, uncomment\r
+    # here and add the restriction for a specific version below\r
+    supported_game_types = (\r
+            'as',\r
+            'ca',\r
+            # 'cq',\r
+            'ctf',\r
+            'cts',\r
+            'dm',\r
+            'dom',\r
+            'ft', 'freezetag',\r
+            'ka', 'keepaway',\r
+            'kh',\r
+            # 'lms',\r
+            'nb', 'nexball',\r
+            # 'rc',\r
+            'rune',\r
+            'tdm',\r
+        )\r
+\r
+    if gametype in supported_game_types:\r
+        is_supported = True\r
+    else:\r
+        is_supported = False\r

 \r
 \r

-    if gametype == 'cts' or gametype == 'ca' or gametype == 'lms':\r
-        flg_supported = False\r
+    # some game types were buggy before revisions, thus this additional filter\r
+    if gametype == 'ca' and version <= 5:\r
+        is_supported = False\r

 \r
 \r

-    return flg_supported\r
+    return is_supported\r

 \r
 \r
 def verify_request(request):\r
 \r
 \r
 def verify_request(request):\r

+    """Verify requests using the d0_blind_id library"""\r
+\r
+    # first determine if we should be verifying or not\r
+    val_verify_requests = request.registry.settings.get('xonstat.verify_requests', 'true')\r
+    if val_verify_requests == "true":\r
+        flg_verify_requests = True\r
+    else:\r
+        flg_verify_requests = False\r
+\r

     try:\r
         (idfp, status) = d0_blind_id_verify(\r
                 sig=request.headers['X-D0-Blind-Id-Detached-Signature'],\r
     try:\r
         (idfp, status) = d0_blind_id_verify(\r
                 sig=request.headers['X-D0-Blind-Id-Detached-Signature'],\r


@@ -30,14 +184,79 @@ def verify_request(request):
                 postdata=request.body)\r
 \r
         log.debug('\nidfp: {0}\nstatus: {1}'.format(idfp, status))\r
                 postdata=request.body)\r
 \r
         log.debug('\nidfp: {0}\nstatus: {1}'.format(idfp, status))\r

-    except: \r
+    except:\r

         idfp = None\r
         status = None\r
 \r
         idfp = None\r
         status = None\r
 \r

+    if flg_verify_requests and not idfp:\r
+        log.debug("ERROR: Unverified request")\r
+        raise pyramid.httpexceptions.HTTPUnauthorized("Unverified request")\r
+\r

     return (idfp, status)\r
 \r
 \r
     return (idfp, status)\r
 \r
 \r

-def has_minimum_real_players(player_events):\r
+def do_precondition_checks(request, game_meta, raw_players):\r
+    """Precondition checks for ALL gametypes.\r
+       These do not require a database connection."""\r
+    if not has_required_metadata(game_meta):\r
+        log.debug("ERROR: Required game meta missing")\r
+        raise pyramid.httpexceptions.HTTPUnprocessableEntity("Missing game meta")\r
+\r
+    try:\r
+        version = int(game_meta['V'])\r
+    except:\r
+        log.debug("ERROR: Required game meta invalid")\r
+        raise pyramid.httpexceptions.HTTPUnprocessableEntity("Invalid game meta")\r
+\r
+    if not is_supported_gametype(game_meta['G'], version):\r
+        log.debug("ERROR: Unsupported gametype")\r
+        raise pyramid.httpexceptions.HTTPOk("OK")\r
+\r
+    if not has_minimum_real_players(request.registry.settings, raw_players):\r
+        log.debug("ERROR: Not enough real players")\r
+        raise pyramid.httpexceptions.HTTPOk("OK")\r
+\r
+    if is_blank_game(game_meta['G'], raw_players):\r
+        log.debug("ERROR: Blank game")\r
+        raise pyramid.httpexceptions.HTTPOk("OK")\r
+\r
+\r
+def is_real_player(events):\r
+    """\r
+    Determines if a given set of events correspond with a non-bot\r
+    """\r
+    if not events['P'].startswith('bot'):\r
+        return True\r
+    else:\r
+        return False\r
+\r
+\r
+def played_in_game(events):\r
+    """\r
+    Determines if a given set of player events correspond with a player who\r
+    played in the game (matches 1 and scoreboardvalid 1)\r
+    """\r
+    if 'matches' in events and 'scoreboardvalid' in events:\r
+        return True\r
+    else:\r
+        return False\r
+\r
+\r
+def num_real_players(player_events):\r
+    """\r
+    Returns the number of real players (those who played\r
+    and are on the scoreboard).\r
+    """\r
+    real_players = 0\r
+\r
+    for events in player_events:\r
+        if is_real_player(events) and played_in_game(events):\r
+            real_players += 1\r
+\r
+    return real_players\r
+\r
+\r
+def has_minimum_real_players(settings, player_events):\r

     """\r
     Determines if the collection of player events has enough "real" players\r
     to store in the database. The minimum setting comes from the config file\r
     """\r
     Determines if the collection of player events has enough "real" players\r
     to store in the database. The minimum setting comes from the config file\r


@@ -45,19 +264,14 @@ def has_minimum_real_players(player_events):
     """\r
     flg_has_min_real_players = True\r
 \r
     """\r
     flg_has_min_real_players = True\r
 \r

-    settings = get_current_registry().settings\r
-    try: \r
+    try:\r

         minimum_required_players = int(\r
                 settings['xonstat.minimum_required_players'])\r
     except:\r
         minimum_required_players = 2\r
 \r
         minimum_required_players = int(\r
                 settings['xonstat.minimum_required_players'])\r
     except:\r
         minimum_required_players = 2\r
 \r

-    real_players = 0\r
-    for events in player_events:\r
-        if is_real_player(events):\r
-            real_players += 1\r
+    real_players = num_real_players(player_events)\r

 \r
 \r

-    #TODO: put this into a config setting in the ini file?\r

     if real_players < minimum_required_players:\r
         flg_has_min_real_players = False\r
 \r
     if real_players < minimum_required_players:\r
         flg_has_min_real_players = False\r
 \r


@@ -74,30 +288,29 @@ def has_required_metadata(metadata):
     if 'T' not in metadata or\\r
         'G' not in metadata or\\r
         'M' not in metadata or\\r
     if 'T' not in metadata or\\r
         'G' not in metadata or\\r
         'M' not in metadata or\\r

+        'I' not in metadata or\\r

         'S' not in metadata:\r
             flg_has_req_metadata = False\r
 \r
     return flg_has_req_metadata\r
 \r
         'S' not in metadata:\r
             flg_has_req_metadata = False\r
 \r
     return flg_has_req_metadata\r
 \r

-    \r
-def is_real_player(events):\r
-    """\r
-    Determines if a given set of player events correspond with a player who\r
-    \r
-    1) is not a bot (P event does not look like a bot)\r
-    2) played in the game (matches 1)\r
-    3) was present at the end of the game (scoreboardvalid 1)\r

 \r
 \r

-    Returns True if the player meets the above conditions, and false otherwise.\r
-    """\r
-    flg_is_real = False\r
+def should_do_weapon_stats(game_type_cd):\r
+    """True of the game type should record weapon stats. False otherwise."""\r
+    if game_type_cd in 'cts':\r
+        return False\r
+    else:\r
+        return True\r

 \r
 \r

-    if not events['P'].startswith('bot'):\r
-        # removing 'joins' here due to bug, but it should be here\r
-        if 'matches' in events and 'scoreboardvalid' in events:\r
-            flg_is_real = True\r

 \r
 \r

-    return flg_is_real\r
+def should_do_elos(game_type_cd):\r
+    """True of the game type should process Elos. False otherwise."""\r
+    elo_game_types = ('duel', 'dm', 'ca', 'ctf', 'tdm', 'ka', 'ft')\r
+\r
+    if game_type_cd in elo_game_types:\r
+        return True\r
+    else:\r
+        return False\r

 \r
 \r
 def register_new_nick(session, player, new_nick):\r
 \r
 \r
 def register_new_nick(session, player, new_nick):\r


@@ -110,15 +323,15 @@ def register_new_nick(session, player, new_nick):
     new_nick - the new nickname\r
     """\r
     # see if that nick already exists\r
     new_nick - the new nickname\r
     """\r
     # see if that nick already exists\r

-    stripped_nick = strip_colors(player.nick)\r
+    stripped_nick = strip_colors(qfont_decode(player.nick))\r

     try:\r
     try:\r

-       player_nick = session.query(PlayerNick).filter_by(\r
-               player_id=player.player_id, stripped_nick=stripped_nick).one()\r
+        player_nick = session.query(PlayerNick).filter_by(\r
+            player_id=player.player_id, stripped_nick=stripped_nick).one()\r

     except NoResultFound, e:\r
     except NoResultFound, e:\r

-           # player_id/stripped_nick not found, create one\r
+        # player_id/stripped_nick not found, create one\r

         # but we don't store "Anonymous Player #N"\r
         if not re.search('^Anonymous Player #\d+$', player.nick):\r
         # but we don't store "Anonymous Player #N"\r
         if not re.search('^Anonymous Player #\d+$', player.nick):\r

-           player_nick = PlayerNick()\r
+            player_nick = PlayerNick()\r

             player_nick.player_id = player.player_id\r
             player_nick.stripped_nick = stripped_nick\r
             player_nick.nick = player.nick\r
             player_nick.player_id = player.player_id\r
             player_nick.stripped_nick = stripped_nick\r
             player_nick.nick = player.nick\r


@@ -126,10 +339,41 @@ def register_new_nick(session, player, new_nick):
 \r
     # We change to the new nick regardless\r
     player.nick = new_nick\r
 \r
     # We change to the new nick regardless\r
     player.nick = new_nick\r

+    player.stripped_nick = strip_colors(qfont_decode(new_nick))\r

     session.add(player)\r
 \r
 \r
     session.add(player)\r
 \r
 \r

-def get_or_create_server(session=None, name=None, hashkey=None):\r
+def update_fastest_cap(session, player_id, game_id,  map_id, captime):\r
+    """\r
+    Check the fastest cap time for the player and map. If there isn't\r
+    one, insert one. If there is, check if the passed time is faster.\r
+    If so, update!\r
+    """\r
+    # we don't record fastest cap times for bots or anonymous players\r
+    if player_id <= 2:\r
+        return\r
+\r
+    # see if a cap entry exists already\r
+    # then check to see if the new captime is faster\r
+    try:\r
+        cur_fastest_cap = session.query(PlayerCaptime).filter_by(\r
+            player_id=player_id, map_id=map_id).one()\r
+\r
+        # current captime is faster, so update\r
+        if captime < cur_fastest_cap.fastest_cap:\r
+            cur_fastest_cap.fastest_cap = captime\r
+            cur_fastest_cap.game_id = game_id\r
+            cur_fastest_cap.create_dt = datetime.datetime.utcnow()\r
+            session.add(cur_fastest_cap)\r
+\r
+    except NoResultFound, e:\r
+        # none exists, so insert\r
+        cur_fastest_cap = PlayerCaptime(player_id, game_id, map_id, captime)\r
+        session.add(cur_fastest_cap)\r
+        session.flush()\r
+\r
+\r
+def get_or_create_server(session, name, hashkey, ip_addr, revision, port):\r

     """\r
     Find a server by name or create one if not found. Parameters:\r
 \r
     """\r
     Find a server by name or create one if not found. Parameters:\r
 \r


@@ -137,31 +381,62 @@ def get_or_create_server(session=None, name=None, hashkey=None):
     name - server name of the server to be found or created\r
     hashkey - server hashkey\r
     """\r
     name - server name of the server to be found or created\r
     hashkey - server hashkey\r
     """\r

-    try:\r
-        # find one by that name, if it exists\r
-        server = session.query(Server).filter_by(name=name).one()\r
-\r
-        # store new hashkey\r
-        if server.hashkey != hashkey:\r
-            server.hashkey = hashkey\r
-            session.add(server)\r
+    server = None\r

 \r
 \r

-        log.debug("Found existing server {0}".format(server.server_id))\r
+    try:\r
+        port = int(port)\r
+    except:\r
+        port = None\r
+\r
+    # finding by hashkey is preferred, but if not we will fall\r
+    # back to using name only, which can result in dupes\r
+    if hashkey is not None:\r
+        servers = session.query(Server).\\r
+            filter_by(hashkey=hashkey).\\r
+            order_by(expr.desc(Server.create_dt)).limit(1).all()\r
+\r
+        if len(servers) > 0:\r
+            server = servers[0]\r
+            log.debug("Found existing server {0} by hashkey ({1})".format(\r
+                server.server_id, server.hashkey))\r
+    else:\r
+        servers = session.query(Server).\\r
+            filter_by(name=name).\\r
+            order_by(expr.desc(Server.create_dt)).limit(1).all()\r

 \r
 \r

-    except MultipleResultsFound, e:\r
-        # multiple found, so also filter by hashkey\r
-        server = session.query(Server).filter_by(name=name).\\r
-                filter_by(hashkey=hashkey).one()\r
-        log.debug("Found existing server {0}".format(server.server_id))\r
+        if len(servers) > 0:\r
+            server = servers[0]\r
+            log.debug("Found existing server {0} by name".format(server.server_id))\r

 \r
 \r

-    except NoResultFound, e:\r
-        # not found, create one\r
+    # still haven't found a server by hashkey or name, so we need to create one\r
+    if server is None:\r

         server = Server(name=name, hashkey=hashkey)\r
         session.add(server)\r
         session.flush()\r
         log.debug("Created server {0} with hashkey {1}".format(\r
             server.server_id, server.hashkey))\r
 \r
         server = Server(name=name, hashkey=hashkey)\r
         session.add(server)\r
         session.flush()\r
         log.debug("Created server {0} with hashkey {1}".format(\r
             server.server_id, server.hashkey))\r
 \r

+    # detect changed fields\r
+    if server.name != name:\r
+        server.name = name\r
+        session.add(server)\r
+\r
+    if server.hashkey != hashkey:\r
+        server.hashkey = hashkey\r
+        session.add(server)\r
+\r
+    if server.ip_addr != ip_addr:\r
+        server.ip_addr = ip_addr\r
+        session.add(server)\r
+\r
+    if server.port != port:\r
+        server.port = port\r
+        session.add(server)\r
+\r
+    if server.revision != revision:\r
+        server.revision = revision\r
+        session.add(server)\r
+\r

     return server\r
 \r
 \r
     return server\r
 \r
 \r


@@ -175,7 +450,7 @@ def get_or_create_map(session=None, name=None):
     try:\r
         # find one by the name, if it exists\r
         gmap = session.query(Map).filter_by(name=name).one()\r
     try:\r
         # find one by the name, if it exists\r
         gmap = session.query(Map).filter_by(name=name).one()\r

-        log.debug("Found map id {0}: {1}".format(gmap.map_id, \r
+        log.debug("Found map id {0}: {1}".format(gmap.map_id,\r

             gmap.name))\r
     except NoResultFound, e:\r
         gmap = Map(name=name)\r
             gmap.name))\r
     except NoResultFound, e:\r
         gmap = Map(name=name)\r


@@ -195,8 +470,8 @@ def get_or_create_map(session=None, name=None):
     return gmap\r
 \r
 \r
     return gmap\r
 \r
 \r

-def create_game(session=None, start_dt=None, game_type_cd=None, \r
-        server_id=None, map_id=None, winner=None):\r
+def create_game(session, start_dt, game_type_cd, server_id, map_id,\r
+        match_id, duration, mod, winner=None):\r

     """\r
     Creates a game. Parameters:\r
 \r
     """\r
     Creates a game. Parameters:\r
 \r


@@ -206,15 +481,37 @@ def create_game(session=None, start_dt=None, game_type_cd=None,
     server_id - server identifier of the server hosting the game\r
     map_id - map on which the game was played\r
     winner - the team id of the team that won\r
     server_id - server identifier of the server hosting the game\r
     map_id - map on which the game was played\r
     winner - the team id of the team that won\r

+    duration - how long the game lasted\r
+    mod - mods in use during the game\r

     """\r
     """\r

-\r
-    game = Game(start_dt=start_dt, game_type_cd=game_type_cd,\r
+    seq = Sequence('games_game_id_seq')\r
+    game_id = session.execute(seq)\r
+    game = Game(game_id=game_id, start_dt=start_dt, game_type_cd=game_type_cd,\r

                 server_id=server_id, map_id=map_id, winner=winner)\r
                 server_id=server_id, map_id=map_id, winner=winner)\r

-    session.add(game)\r
-    session.flush()\r
-    log.debug("Created game id {0} on server {1}, map {2} at \\r
-            {3}".format(game.game_id, \r
-                server_id, map_id, start_dt))\r
+    game.match_id = match_id\r
+    game.mod = mod[:64]\r
+\r
+    try:\r
+        game.duration = datetime.timedelta(seconds=int(round(float(duration))))\r
+    except:\r
+        pass\r
+\r
+    try:\r
+        session.query(Game).filter(Game.server_id==server_id).\\r
+                filter(Game.match_id==match_id).one()\r
+\r
+        log.debug("Error: game with same server and match_id found! Ignoring.")\r
+\r
+        # if a game under the same server and match_id found,\r
+        # this is a duplicate game and can be ignored\r
+        raise pyramid.httpexceptions.HTTPOk('OK')\r
+    except NoResultFound, e:\r
+        # server_id/match_id combination not found. game is ok to insert\r
+        session.add(game)\r
+        session.flush()\r
+        log.debug("Created game id {0} on server {1}, map {2} at \\r
+                {3}".format(game.game_id,\r
+                    server_id, map_id, start_dt))\r

 \r
     return game\r
 \r
 \r
     return game\r
 \r


@@ -229,7 +526,7 @@ def get_or_create_player(session=None, hashkey=None, nick=None):
     nick - nick of the player (in case of a first time create)\r
     """\r
     # if we have a bot\r
     nick - nick of the player (in case of a first time create)\r
     """\r
     # if we have a bot\r

-    if re.search('^bot#\d+$', hashkey):\r
+    if re.search('^bot#\d+$', hashkey) or re.search('^bot#\d+#', hashkey):\r

         player = session.query(Player).filter_by(player_id=1).one()\r
     # if we have an untracked player\r
     elif re.search('^player#\d+$', hashkey):\r
         player = session.query(Player).filter_by(player_id=1).one()\r
     # if we have an untracked player\r
     elif re.search('^player#\d+$', hashkey):\r


@@ -239,152 +536,223 @@ def get_or_create_player(session=None, hashkey=None, nick=None):
         # see if the player is already in the database\r
         # if not, create one and the hashkey along with it\r
         try:\r
         # see if the player is already in the database\r
         # if not, create one and the hashkey along with it\r
         try:\r

-            hashkey = session.query(Hashkey).filter_by(\r
+            hk = session.query(Hashkey).filter_by(\r

                     hashkey=hashkey).one()\r
             player = session.query(Player).filter_by(\r
                     hashkey=hashkey).one()\r
             player = session.query(Player).filter_by(\r

-                    player_id=hashkey.player_id).one()\r
+                    player_id=hk.player_id).one()\r

             log.debug("Found existing player {0} with hashkey {1}".format(\r
             log.debug("Found existing player {0} with hashkey {1}".format(\r

-                player.player_id, hashkey.hashkey))\r
+                player.player_id, hashkey))\r

         except:\r
             player = Player()\r
             session.add(player)\r
             session.flush()\r
 \r
         except:\r
             player = Player()\r
             session.add(player)\r
             session.flush()\r
 \r

-           # if nick is given to us, use it. If not, use "Anonymous Player"\r
+            # if nick is given to us, use it. If not, use "Anonymous Player"\r

             # with a suffix added for uniqueness.\r
             if nick:\r
                 player.nick = nick[:128]\r
             # with a suffix added for uniqueness.\r
             if nick:\r
                 player.nick = nick[:128]\r

-           else:\r
+                player.stripped_nick = strip_colors(qfont_decode(nick[:128]))\r
+            else:\r

                 player.nick = "Anonymous Player #{0}".format(player.player_id)\r
                 player.nick = "Anonymous Player #{0}".format(player.player_id)\r

+                player.stripped_nick = player.nick\r

 \r
 \r

-            hashkey = Hashkey(player_id=player.player_id, hashkey=hashkey)\r
-            session.add(hashkey)\r
+            hk = Hashkey(player_id=player.player_id, hashkey=hashkey)\r
+            session.add(hk)\r

             log.debug("Created player {0} ({2}) with hashkey {1}".format(\r
             log.debug("Created player {0} ({2}) with hashkey {1}".format(\r

-                player.player_id, hashkey.hashkey, player.nick.encode('utf-8')))\r
+                player.player_id, hashkey, player.nick.encode('utf-8')))\r

 \r
     return player\r
 \r
 \r
     return player\r
 \r

-def create_player_game_stat(session=None, player=None, \r
-        game=None, player_events=None):\r
-    """\r
-    Creates game statistics for a given player in a given game. Parameters:\r

 \r
 \r

-    session - SQLAlchemy session factory\r
-    player - Player record of the player who owns the stats\r
-    game - Game record for the game to which the stats pertain\r
-    player_events - dictionary for the actual stats that need to be transformed\r
-    """\r
+def create_default_game_stat(session, game_type_cd):\r
+    """Creates a blanked-out pgstat record for the given game type"""\r
+\r
+    # this is what we have to do to get partitioned records in - grab the\r
+    # sequence value first, then insert using the explicit ID (vs autogenerate)\r
+    seq = Sequence('player_game_stats_player_game_stat_id_seq')\r
+    pgstat_id = session.execute(seq)\r
+    pgstat = PlayerGameStat(player_game_stat_id=pgstat_id,\r
+            create_dt=datetime.datetime.utcnow())\r

 \r
 \r

-    # in here setup default values (e.g. if game type is CTF then\r
-    # set kills=0, score=0, captures=0, pickups=0, fckills=0, etc\r
-    # TODO: use game's create date here instead of now()\r
-    pgstat = PlayerGameStat(create_dt=datetime.datetime.now())\r
+    if game_type_cd == 'as':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = pgstat.collects = 0\r

 \r
 \r

-    # set player id from player record\r
-    pgstat.player_id = player.player_id\r
+    if game_type_cd in 'ca' 'dm' 'duel' 'rune' 'tdm':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = 0\r

 \r
 \r

-    #set game id from game record\r
-    pgstat.game_id = game.game_id\r
+    if game_type_cd == 'cq':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = pgstat.captures = 0\r
+        pgstat.drops = 0\r

 \r
 \r

-    # all games have a score\r
-    pgstat.score = 0\r
+    if game_type_cd == 'ctf':\r
+        pgstat.kills = pgstat.captures = pgstat.pickups = pgstat.drops = 0\r
+        pgstat.returns = pgstat.carrier_frags = 0\r

 \r
 \r

-    if game.game_type_cd == 'dm':\r
-        pgstat.kills = 0\r
+    if game_type_cd == 'cts':\r

         pgstat.deaths = 0\r
         pgstat.deaths = 0\r

-        pgstat.suicides = 0\r
-    elif game.game_type_cd == 'ctf':\r
-        pgstat.kills = 0\r
-        pgstat.captures = 0\r
-        pgstat.pickups = 0\r
+\r
+    if game_type_cd == 'dom':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = pgstat.pickups = 0\r

         pgstat.drops = 0\r
         pgstat.drops = 0\r

-        pgstat.returns = 0\r
+\r
+    if game_type_cd == 'ft':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = pgstat.revivals = 0\r
+\r
+    if game_type_cd == 'ka':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = pgstat.pickups = 0\r
+        pgstat.carrier_frags = 0\r
+        pgstat.time = datetime.timedelta(seconds=0)\r
+\r
+    if game_type_cd == 'kh':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = pgstat.pickups = 0\r
+        pgstat.captures = pgstat.drops = pgstat.pushes = pgstat.destroys = 0\r

         pgstat.carrier_frags = 0\r
 \r
         pgstat.carrier_frags = 0\r
 \r

-    for (key,value) in player_events.items():\r
-        if key == 'n': pgstat.nick = value[:128]\r
-        if key == 't': pgstat.team = value\r
-        if key == 'rank': pgstat.rank = value\r
-        if key == 'alivetime': \r
-            pgstat.alivetime = datetime.timedelta(seconds=int(round(float(value))))\r
-        if key == 'scoreboard-drops': pgstat.drops = value\r
-        if key == 'scoreboard-returns': pgstat.returns = value\r
-        if key == 'scoreboard-fckills': pgstat.carrier_frags = value\r
-        if key == 'scoreboard-pickups': pgstat.pickups = value\r
-        if key == 'scoreboard-caps': pgstat.captures = value\r
-        if key == 'scoreboard-score': pgstat.score = value\r
-        if key == 'scoreboard-deaths': pgstat.deaths = value\r
-        if key == 'scoreboard-kills': pgstat.kills = value\r
-        if key == 'scoreboard-suicides': pgstat.suicides = value\r
-\r
-    # check to see if we had a name, and if \r
-    # not use the name from the player id\r
-    if pgstat.nick == None:\r
-        pgstat.nick = player.nick\r
-\r
-    # if the nick we end up with is different from the one in the\r
-    # player record, change the nick to reflect the new value\r
-    if pgstat.nick != player.nick and player.player_id > 2:\r
+    if game_type_cd == 'lms':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = pgstat.lives = 0\r
+\r
+    if game_type_cd == 'nb':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = pgstat.captures = 0\r
+        pgstat.drops = 0\r
+\r
+    if game_type_cd == 'rc':\r
+        pgstat.kills = pgstat.deaths = pgstat.suicides = pgstat.laps = 0\r
+\r
+    return pgstat\r
+\r
+\r
+def create_game_stat(session, game_meta, game, server, gmap, player, events):\r
+    """Game stats handler for all game types"""\r
+\r
+    game_type_cd = game.game_type_cd\r
+\r
+    pgstat = create_default_game_stat(session, game_type_cd)\r
+\r
+    # these fields should be on every pgstat record\r
+    pgstat.game_id       = game.game_id\r
+    pgstat.player_id     = player.player_id\r
+    pgstat.nick          = events.get('n', 'Anonymous Player')[:128]\r
+    pgstat.stripped_nick = strip_colors(qfont_decode(pgstat.nick))\r
+    pgstat.score         = int(round(float(events.get('scoreboard-score', 0))))\r
+    pgstat.alivetime     = datetime.timedelta(seconds=int(round(float(events.get('alivetime', 0.0)))))\r
+    pgstat.rank          = int(events.get('rank', None))\r
+    pgstat.scoreboardpos = int(events.get('scoreboardpos', pgstat.rank))\r
+\r
+    if pgstat.nick != player.nick \\r
+            and player.player_id > 2 \\r
+            and pgstat.nick != 'Anonymous Player':\r

         register_new_nick(session, player, pgstat.nick)\r
 \r
         register_new_nick(session, player, pgstat.nick)\r
 \r

-    # if the player is ranked #1 and it is a team game, set the game's winner\r
-    # to be the team of that player\r
-    # FIXME: this is a hack, should be using the 'W' field (not present)\r
-    if pgstat.rank == '1' and pgstat.team:\r
+    wins = False\r
+\r
+    # gametype-specific stuff is handled here. if passed to us, we store it\r
+    for (key,value) in events.items():\r
+        if key == 'wins': wins = True\r
+        if key == 't': pgstat.team = int(value)\r
+\r
+        if key == 'scoreboard-drops': pgstat.drops = int(value)\r
+        if key == 'scoreboard-returns': pgstat.returns = int(value)\r
+        if key == 'scoreboard-fckills': pgstat.carrier_frags = int(value)\r
+        if key == 'scoreboard-pickups': pgstat.pickups = int(value)\r
+        if key == 'scoreboard-caps': pgstat.captures = int(value)\r
+        if key == 'scoreboard-score': pgstat.score = int(round(float(value)))\r
+        if key == 'scoreboard-deaths': pgstat.deaths = int(value)\r
+        if key == 'scoreboard-kills': pgstat.kills = int(value)\r
+        if key == 'scoreboard-suicides': pgstat.suicides = int(value)\r
+        if key == 'scoreboard-objectives': pgstat.collects = int(value)\r
+        if key == 'scoreboard-captured': pgstat.captures = int(value)\r
+        if key == 'scoreboard-released': pgstat.drops = int(value)\r
+        if key == 'scoreboard-fastest':\r
+            pgstat.fastest = datetime.timedelta(seconds=float(value)/100)\r
+        if key == 'scoreboard-takes': pgstat.pickups = int(value)\r
+        if key == 'scoreboard-ticks': pgstat.drops = int(value)\r
+        if key == 'scoreboard-revivals': pgstat.revivals = int(value)\r
+        if key == 'scoreboard-bctime':\r
+            pgstat.time = datetime.timedelta(seconds=int(value))\r
+        if key == 'scoreboard-bckills': pgstat.carrier_frags = int(value)\r
+        if key == 'scoreboard-losses': pgstat.drops = int(value)\r
+        if key == 'scoreboard-pushes': pgstat.pushes = int(value)\r
+        if key == 'scoreboard-destroyed': pgstat.destroys = int(value)\r
+        if key == 'scoreboard-kckills': pgstat.carrier_frags = int(value)\r
+        if key == 'scoreboard-lives': pgstat.lives = int(value)\r
+        if key == 'scoreboard-goals': pgstat.captures = int(value)\r
+        if key == 'scoreboard-faults': pgstat.drops = int(value)\r
+        if key == 'scoreboard-laps': pgstat.laps = int(value)\r
+\r
+        if key == 'avglatency': pgstat.avg_latency = float(value)\r
+        if key == 'scoreboard-captime':\r
+            pgstat.fastest = datetime.timedelta(seconds=float(value)/100)\r
+            if game.game_type_cd == 'ctf':\r
+                update_fastest_cap(session, player.player_id, game.game_id,\r
+                        gmap.map_id, pgstat.fastest)\r
+\r
+    # there is no "winning team" field, so we have to derive it\r
+    if wins and pgstat.team is not None and game.winner is None:\r

         game.winner = pgstat.team\r
         session.add(game)\r
 \r
     session.add(pgstat)\r
         game.winner = pgstat.team\r
         session.add(game)\r
 \r
     session.add(pgstat)\r

-    session.flush()\r

 \r
     return pgstat\r
 \r
 \r
 \r
     return pgstat\r
 \r
 \r

-def create_player_weapon_stats(session=None, player=None, \r
-        game=None, pgstat=None, player_events=None):\r
-    """\r
-    Creates accuracy records for each weapon used by a given player in a\r
-    given game. Parameters:\r
-\r
-    session - SQLAlchemy session factory object\r
-    player - Player record who owns the weapon stats\r
-    game - Game record in which the stats were created\r
-    pgstat - Corresponding PlayerGameStat record for these weapon stats\r
-    player_events - dictionary containing the raw weapon values that need to be\r
-        transformed\r
-    """\r
+def create_weapon_stats(session, game_meta, game, player, pgstat, events):\r
+    """Weapon stats handler for all game types"""\r

     pwstats = []\r
 \r
     pwstats = []\r
 \r

-    for (key,value) in player_events.items():\r
+    # Version 1 of stats submissions doubled the data sent.\r
+    # To counteract this we divide the data by 2 only for\r
+    # POSTs coming from version 1.\r
+    try:\r
+        version = int(game_meta['V'])\r
+        if version == 1:\r
+            is_doubled = True\r
+            log.debug('NOTICE: found a version 1 request, halving the weapon stats...')\r
+        else:\r
+            is_doubled = False\r
+    except:\r
+        is_doubled = False\r
+\r
+    for (key,value) in events.items():\r

         matched = re.search("acc-(.*?)-cnt-fired", key)\r
         if matched:\r
             weapon_cd = matched.group(1)\r
         matched = re.search("acc-(.*?)-cnt-fired", key)\r
         if matched:\r
             weapon_cd = matched.group(1)\r

+            seq = Sequence('player_weapon_stats_player_weapon_stats_id_seq')\r
+            pwstat_id = session.execute(seq)\r

             pwstat = PlayerWeaponStat()\r
             pwstat = PlayerWeaponStat()\r

+            pwstat.player_weapon_stats_id = pwstat_id\r

             pwstat.player_id = player.player_id\r
             pwstat.game_id = game.game_id\r
             pwstat.player_game_stat_id = pgstat.player_game_stat_id\r
             pwstat.weapon_cd = weapon_cd\r
 \r
             pwstat.player_id = player.player_id\r
             pwstat.game_id = game.game_id\r
             pwstat.player_game_stat_id = pgstat.player_game_stat_id\r
             pwstat.weapon_cd = weapon_cd\r
 \r

-            if 'n' in player_events:\r
-                pwstat.nick = player_events['n']\r
+            if 'n' in events:\r
+                pwstat.nick = events['n']\r

             else:\r
             else:\r

-                pwstat.nick = player_events['P']\r
+                pwstat.nick = events['P']\r

 \r
 \r

-            if 'acc-' + weapon_cd + '-cnt-fired' in player_events:\r
+            if 'acc-' + weapon_cd + '-cnt-fired' in events:\r

                 pwstat.fired = int(round(float(\r
                 pwstat.fired = int(round(float(\r

-                        player_events['acc-' + weapon_cd + '-cnt-fired'])))\r
-            if 'acc-' + weapon_cd + '-fired' in player_events:\r
+                        events['acc-' + weapon_cd + '-cnt-fired'])))\r
+            if 'acc-' + weapon_cd + '-fired' in events:\r

                 pwstat.max = int(round(float(\r
                 pwstat.max = int(round(float(\r

-                        player_events['acc-' + weapon_cd + '-fired'])))\r
-            if 'acc-' + weapon_cd + '-cnt-hit' in player_events:\r
+                        events['acc-' + weapon_cd + '-fired'])))\r
+            if 'acc-' + weapon_cd + '-cnt-hit' in events:\r

                 pwstat.hit = int(round(float(\r
                 pwstat.hit = int(round(float(\r

-                        player_events['acc-' + weapon_cd + '-cnt-hit'])))\r
-            if 'acc-' + weapon_cd + '-hit' in player_events:\r
+                        events['acc-' + weapon_cd + '-cnt-hit'])))\r
+            if 'acc-' + weapon_cd + '-hit' in events:\r

                 pwstat.actual = int(round(float(\r
                 pwstat.actual = int(round(float(\r

-                        player_events['acc-' + weapon_cd + '-hit'])))\r
-            if 'acc-' + weapon_cd + '-frags' in player_events:\r
+                        events['acc-' + weapon_cd + '-hit'])))\r
+            if 'acc-' + weapon_cd + '-frags' in events:\r

                 pwstat.frags = int(round(float(\r
                 pwstat.frags = int(round(float(\r

-                        player_events['acc-' + weapon_cd + '-frags'])))\r
+                        events['acc-' + weapon_cd + '-frags'])))\r
+\r
+            if is_doubled:\r
+                pwstat.fired = pwstat.fired/2\r
+                pwstat.max = pwstat.max/2\r
+                pwstat.hit = pwstat.hit/2\r
+                pwstat.actual = pwstat.actual/2\r
+                pwstat.frags = pwstat.frags/2\r

 \r
             session.add(pwstat)\r
             pwstats.append(pwstat)\r
 \r
             session.add(pwstat)\r
             pwstats.append(pwstat)\r


@@ -392,131 +760,92 @@ def create_player_weapon_stats(session=None, player=None,
     return pwstats\r
 \r
 \r
     return pwstats\r
 \r
 \r

-def parse_body(request):\r
-    """\r
-    Parses the POST request body for a stats submission\r
-    """\r
-    # storage vars for the request body\r
-    game_meta = {}\r
-    player_events = {}\r
-    current_team = None\r
-    players = []\r
-    \r
-    log.debug("----- BEGIN REQUEST BODY -----")\r
-    log.debug(request.body)\r
-    log.debug("----- END REQUEST BODY -----")\r
+def create_elos(session, game):\r
+    """Elo handler for all game types."""\r
+    try:\r
+        process_elos(game, session)\r
+    except Exception as e:\r
+        log.debug('Error (non-fatal): elo processing failed.')\r

 \r
 \r

-    for line in request.body.split('\n'):\r
-        try:\r
-            (key, value) = line.strip().split(' ', 1)\r

 \r
 \r

-            # Server (S) and Nick (n) fields can have international characters.\r
-            # We encode these as UTF-8.\r
-            if key in 'S' 'n':\r
-                value = unicode(value, 'utf-8')\r
-    \r
-            if key in 'V' 'T' 'G' 'M' 'S' 'C' 'R' 'W':\r
-                game_meta[key] = value\r
+def submit_stats(request):\r
+    """\r
+    Entry handler for POST stats submissions.\r
+    """\r
+    try:\r
+        # placeholder for the actual session\r
+        session = None\r

 \r
 \r

-            if key == 'P':\r
-                # if we were working on a player record already, append\r
-                # it and work on a new one (only set team info)\r
-                if len(player_events) != 0:\r
-                    players.append(player_events)\r
-                    player_events = {}\r
-    \r
-                player_events[key] = value\r
+        log.debug("\n----- BEGIN REQUEST BODY -----\n" + request.body +\r
+                "----- END REQUEST BODY -----\n\n")\r

 \r
 \r

-            if key == 'e':\r
-                (subkey, subvalue) = value.split(' ', 1)\r
-                player_events[subkey] = subvalue\r
-            if key == 'n':\r
-                player_events[key] = value\r
-            if key == 't':\r
-                player_events[key] = value\r
-        except:\r
-            # no key/value pair - move on to the next line\r
-            pass\r
-    \r
-    # add the last player we were working on\r
-    if len(player_events) > 0:\r
-        players.append(player_events)\r
-\r
-    return (game_meta, players)\r
+        (idfp, status) = verify_request(request)\r
+        (game_meta, raw_players, raw_teams) = parse_stats_submission(request.body)\r
+        revision = game_meta.get('R', 'unknown')\r
+        duration = game_meta.get('D', None)\r

 \r
 \r

+        # only players present at the end of the match are eligible for stats\r
+        raw_players = filter(played_in_game, raw_players)\r

 \r
 \r

-def create_player_stats(session=None, player=None, game=None, \r
-        player_events=None):\r
-    """\r
-    Creates player game and weapon stats according to what type of player\r
-    """\r
-    pgstat = create_player_game_stat(session=session, \r
-        player=player, game=game, player_events=player_events)\r
+        do_precondition_checks(request, game_meta, raw_players)\r

 \r
 \r

-    #TODO: put this into a config setting in the ini file?\r
-    if not re.search('^bot#\d+$', player_events['P']):\r
-        create_player_weapon_stats(session=session, \r
-            player=player, game=game, pgstat=pgstat,\r
-            player_events=player_events)\r
-    \r
+        # the "duel" gametype is fake\r
+        if len(raw_players) == 2 \\r
+            and num_real_players(raw_players) == 2 \\r
+            and game_meta['G'] == 'dm':\r
+            game_meta['G'] = 'duel'\r

 \r
 \r

-def stats_submit(request):\r
-    """\r
-    Entry handler for POST stats submissions.\r
-    """\r
-    try:\r
+        #----------------------------------------------------------------------\r
+        # Actual setup (inserts/updates) below here\r
+        #----------------------------------------------------------------------\r

         session = DBSession()\r
 \r
         session = DBSession()\r
 \r

-        (idfp, status) = verify_request(request)\r
-        if not idfp:\r
-            raise pyramid.httpexceptions.HTTPUnauthorized\r
-\r
-        (game_meta, players) = parse_body(request)  \r
-    \r
-        if not has_required_metadata(game_meta):\r
-            log.debug("Required game meta fields missing. "\\r
-                    "Can't continue.")\r
-            raise pyramid.exceptions.HTTPUnprocessableEntity\r
-   \r
-        if not is_supported_gametype(game_meta['G']):\r
-            raise pyramid.httpexceptions.HTTPOk\r
-     \r
-        if not has_minimum_real_players(players):\r
-            log.debug("The number of real players is below the minimum. " + \r
-                "Stats will be ignored.")\r
-            raise pyramid.httpexceptions.HTTPOk\r
-\r
-        server = get_or_create_server(session=session, hashkey=idfp, \r
-                name=game_meta['S'])\r
-\r
-        gmap = get_or_create_map(session=session, name=game_meta['M'])\r
-\r
-        game = create_game(session=session, \r
-                start_dt=datetime.datetime(\r
-                    *time.gmtime(float(game_meta['T']))[:6]), \r
-                server_id=server.server_id, game_type_cd=game_meta['G'], \r
-                map_id=gmap.map_id)\r
-    \r
-        # find or create a record for each player\r
-        # and add stats for each if they were present at the end\r
-        # of the game\r
-        for player_events in players:\r
-            if 'n' in player_events:\r
-                nick = player_events['n']\r
-            else:\r
-                nick = None\r
-\r
-            if 'matches' in player_events and 'scoreboardvalid' \\r
-                    in player_events:\r
-                player = get_or_create_player(session=session, \r
-                    hashkey=player_events['P'], nick=nick)\r
-                log.debug('Creating stats for %s' % player_events['P'])\r
-                create_player_stats(session=session, player=player, game=game, \r
-                        player_events=player_events)\r
-    \r
+        game_type_cd = game_meta['G']\r
+\r
+        # All game types create Game, Server, Map, and Player records\r
+        # the same way.\r
+        server = get_or_create_server(\r
+                session  = session,\r
+                hashkey  = idfp,\r
+                name     = game_meta['S'],\r
+                revision = revision,\r
+                ip_addr  = get_remote_addr(request),\r
+                port     = game_meta.get('U', None))\r
+\r
+        gmap = get_or_create_map(\r
+                session = session,\r
+                name    = game_meta['M'])\r
+\r
+        game = create_game(\r
+                session      = session,\r
+                start_dt     = datetime.datetime.utcnow(),\r
+                server_id    = server.server_id,\r
+                game_type_cd = game_type_cd,\r
+                map_id       = gmap.map_id,\r
+                match_id     = game_meta['I'],\r
+                duration     = duration,\r
+                mod          = game_meta.get('O', None))\r
+\r
+        for events in raw_players:\r
+            player = get_or_create_player(\r
+                session = session,\r
+                hashkey = events['P'],\r
+                nick    = events.get('n', None))\r
+\r
+            pgstat = create_game_stat(session, game_meta, game, server,\r
+                    gmap, player, events)\r
+\r
+            if should_do_weapon_stats(game_type_cd) and player.player_id > 1:\r
+                pwstats = create_weapon_stats(session, game_meta, game, player,\r
+                        pgstat, events)\r
+\r
+        if should_do_elos(game_type_cd):\r
+            create_elos(session, game)\r
+\r

         session.commit()\r
         log.debug('Success! Stats recorded.')\r
         return Response('200 OK')\r
     except Exception as e:\r
         session.commit()\r
         log.debug('Success! Stats recorded.')\r
         return Response('200 OK')\r
     except Exception as e:\r

-        session.rollback()\r
-        raise e\r
+        if session:\r
+            session.rollback()\r
+        return e\r