Always set the returned Elo dict. Small PEP8 changes.

[xonotic/xonstat.git] / xonstat / views / submission.py

diff --git a/xonstat/views/submission.py b/xonstat/views/submission.py
index 40f735d01e99395548ce9834f53d45a0782699b9..048a48863e2907b18913e620228b71fcd05d1148 100644 (file)
--- a/xonstat/views/submission.py
+++ b/xonstat/views/submission.py
@@ -6,7 +6,7 @@ import re
 
 import pyramid.httpexceptions
 from sqlalchemy import Sequence
 
 import pyramid.httpexceptions
 from sqlalchemy import Sequence

-from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound
+from sqlalchemy.orm.exc import NoResultFound

 from xonstat.elo import EloProcessor
 from xonstat.models import DBSession, Server, Map, Game, PlayerGameStat, PlayerWeaponStat
 from xonstat.models import PlayerRank, PlayerCaptime
 from xonstat.elo import EloProcessor
 from xonstat.models import DBSession, Server, Map, Game, PlayerGameStat, PlayerWeaponStat
 from xonstat.models import PlayerRank, PlayerCaptime


@@ -16,27 +16,6 @@ from xonstat.util import strip_colors, qfont_decode, verify_request, weapon_map
 log = logging.getLogger(__name__)
 
 
 log = logging.getLogger(__name__)
 
 

-def is_real_player(events):
-    """
-    Determines if a given set of events correspond with a non-bot
-    """
-    if not events['P'].startswith('bot'):
-        return True
-    else:
-        return False
-
-
-def played_in_game(events):
-    """
-    Determines if a given set of player events correspond with a player who
-    played in the game (matches 1 and scoreboardvalid 1)
-    """
-    if 'matches' in events and 'scoreboardvalid' in events:
-        return True
-    else:
-        return False
-
-

 class Submission(object):
     """Parses an incoming POST request for stats submissions."""
 
 class Submission(object):
     """Parses an incoming POST request for stats submissions."""
 


@@ -125,11 +104,24 @@ class Submission(object):
         except:
             return None, None
 
         except:
             return None, None
 

-    def check_for_new_weapon_fired(self, sub_key):
-        """Checks if a given weapon fired event is a new one for the match."""
-        weapon = sub_key.split("-")[1]
-        if weapon not in self.weapons:
-            self.weapons.add(weapon)
+    def add_weapon_fired(self, sub_key):
+        """Adds a weapon to the set of weapons fired during the match (a set)."""
+        self.weapons.add(sub_key.split("-")[1])
+
+    @staticmethod
+    def is_human_player(player):
+        """
+        Determines if a given set of events correspond with a non-bot
+        """
+        return not player['P'].startswith('bot')
+
+    @staticmethod
+    def played_in_game(player):
+        """
+        Determines if a given set of player events correspond with a player who
+        played in the game (matches 1 and scoreboardvalid 1)
+        """
+        return 'matches' in player and 'scoreboardvalid' in player

 
     def parse_player(self, key, pid):
         """Construct a player events listing from the submission."""
 
     def parse_player(self, key, pid):
         """Construct a player events listing from the submission."""


@@ -154,7 +146,7 @@ class Submission(object):
 
                 if sub_key.endswith("cnt-fired"):
                     player_fired_weapon = True
 
                 if sub_key.endswith("cnt-fired"):
                     player_fired_weapon = True

-                    self.check_for_new_weapon_fired(sub_key)
+                    self.add_weapon_fired(sub_key)

                 elif sub_key == 'scoreboard-score' and int(sub_value) != 0:
                     player_nonzero_score = True
                 elif sub_key == 'scoreboard-fastest':
                 elif sub_key == 'scoreboard-score' and int(sub_value) != 0:
                     player_nonzero_score = True
                 elif sub_key == 'scoreboard-fastest':


@@ -168,8 +160,8 @@ class Submission(object):
                 self.q.appendleft("{} {}".format(key, value))
                 break
 
                 self.q.appendleft("{} {}".format(key, value))
                 break
 

-        played = played_in_game(player)
-        human = is_real_player(player)
+        played = self.played_in_game(player)
+        human = self.is_human_player(player)

 
         if played and human:
             self.humans.append(player)
 
         if played and human:
             self.humans.append(player)


@@ -185,8 +177,8 @@ class Submission(object):
 
         elif played and not human:
             self.bots.append(player)
 
         elif played and not human:
             self.bots.append(player)

-        else:
-            self.players.append(player)
+
+        self.players.append(player)

 
     def parse_team(self, key, tid):
         """Construct a team events listing from the submission."""
 
     def parse_team(self, key, tid):
         """Construct a team events listing from the submission."""


@@ -237,10 +229,16 @@ class Submission(object):
 
         return self
 
 
         return self
 

+    def __repr__(self):
+        """Debugging representation of a submission."""
+        return "game_type_cd: {}, mod: {}, players: {}, humans: {}, bots: {}, weapons: {}".format(
+            self.game_type_cd, self.mod, len(self.players), len(self.humans), len(self.bots),
+            self.weapons)
+

 
 def elo_submission_category(submission):
     """Determines the Elo category purely by what is in the submission data."""
 
 def elo_submission_category(submission):
     """Determines the Elo category purely by what is in the submission data."""

-    mod = submission.meta.get("O", "None")
+    mod = submission.mod

 
     vanilla_allowed_weapons = {"shotgun", "devastator", "blaster", "mortar", "vortex", "electro",
                                "arc", "hagar", "crylink", "machinegun"}
 
     vanilla_allowed_weapons = {"shotgun", "devastator", "blaster", "mortar", "vortex", "electro",
                                "arc", "hagar", "crylink", "machinegun"}


@@ -262,79 +260,9 @@ def elo_submission_category(submission):
     return "general"
 
 
     return "general"
 
 

-def parse_stats_submission(body):
-    """
-    Parses the POST request body for a stats submission
+def is_blank_game(submission):

     """
     """

-    # storage vars for the request body
-    game_meta = {}
-    events = {}
-    players = []
-    teams = []
-
-    # we're not in either stanza to start
-    in_P = in_Q = False
-
-    for line in body.split('\n'):
-        try:
-            (key, value) = line.strip().split(' ', 1)
-
-            # Server (S) and Nick (n) fields can have international characters.
-            if key in 'S' 'n':
-                value = unicode(value, 'utf-8')
-
-            if key not in 'P' 'Q' 'n' 'e' 't' 'i':
-                game_meta[key] = value
-
-            if key == 'Q' or key == 'P':
-                #log.debug('Found a {0}'.format(key))
-                #log.debug('in_Q: {0}'.format(in_Q))
-                #log.debug('in_P: {0}'.format(in_P))
-                #log.debug('events: {0}'.format(events))
-
-                # check where we were before and append events accordingly
-                if in_Q and len(events) > 0:
-                    #log.debug('creating a team (Q) entry')
-                    teams.append(events)
-                    events = {}
-                elif in_P and len(events) > 0:
-                    #log.debug('creating a player (P) entry')
-                    players.append(events)
-                    events = {}
-
-                if key == 'P':
-                    #log.debug('key == P')
-                    in_P = True
-                    in_Q = False
-                elif key == 'Q':
-                    #log.debug('key == Q')
-                    in_P = False
-                    in_Q = True
-
-                events[key] = value
-
-            if key == 'e':
-                (subkey, subvalue) = value.split(' ', 1)
-                events[subkey] = subvalue
-            if key == 'n':
-                events[key] = value
-            if key == 't':
-                events[key] = value
-        except:
-            # no key/value pair - move on to the next line
-            pass
-
-    # add the last entity we were working on
-    if in_P and len(events) > 0:
-        players.append(events)
-    elif in_Q and len(events) > 0:
-        teams.append(events)
-
-    return (game_meta, players, teams)
-
-
-def is_blank_game(gametype, players):
-    """Determine if this is a blank game or not. A blank game is either:
+    Determine if this is a blank game or not. A blank game is either:

 
     1) a match that ended in the warmup stage, where accuracy events are not
     present (for non-CTS games)
 
     1) a match that ended in the warmup stage, where accuracy events are not
     present (for non-CTS games)


@@ -351,40 +279,24 @@ def is_blank_game(gametype, players):
 
     1) a match in which no player made a positive or negative score
     """
 
     1) a match in which no player made a positive or negative score
     """

-    r = re.compile(r'acc-.*-cnt-fired')
-    flg_nonzero_score = False
-    flg_acc_events = False
-    flg_fastest_lap = False
-
-    for events in players:
-        if is_real_player(events) and played_in_game(events):
-            for (key,value) in events.items():
-                if key == 'scoreboard-score' and value != 0:
-                    flg_nonzero_score = True
-                if r.search(key):
-                    flg_acc_events = True
-                if key == 'scoreboard-fastest':
-                    flg_fastest_lap = True
-
-    if gametype == 'cts':
-        return not flg_fastest_lap
-    elif gametype == 'nb':
-        return not flg_nonzero_score
+    if submission.game_type_cd == 'cts':
+        return not submission.human_fastest
+    elif submission.game_type_cd == 'nb':
+        return not submission.human_nonzero_score

     else:
     else:

-        return not (flg_nonzero_score and flg_acc_events)
+        return not (submission.human_nonzero_score and submission.human_fired_weapon)

 
 
 
 

-def get_remote_addr(request):
-    """Get the Xonotic server's IP address"""
-    if 'X-Forwarded-For' in request.headers:
-        return request.headers['X-Forwarded-For']
-    else:
-        return request.remote_addr
+def has_required_metadata(submission):
+    """Determines if a submission has all the required metadata fields."""
+    return (submission.game_type_cd is not None
+            and submission.map_name is not None
+            and submission.match_id is not None
+            and submission.server_name is not None)

 
 
 
 

-def is_supported_gametype(gametype, version):
-    """Whether a gametype is supported or not"""
-    is_supported = False
+def is_supported_gametype(submission):
+    """Determines if a submission is of a valid and supported game type."""

 
     # if the type can be supported, but with version constraints, uncomment
     # here and add the restriction for a specific version below
 
     # if the type can be supported, but with version constraints, uncomment
     # here and add the restriction for a specific version below


@@ -407,22 +319,31 @@ def is_supported_gametype(gametype, version):
             'tdm',
         )
 
             'tdm',
         )
 

-    if gametype in supported_game_types:
-        is_supported = True
-    else:
-        is_supported = False
+    is_supported = submission.game_type_cd in supported_game_types

 
     # some game types were buggy before revisions, thus this additional filter
 
     # some game types were buggy before revisions, thus this additional filter

-    if gametype == 'ca' and version <= 5:
+    if submission.game_type_cd == 'ca' and submission.version <= 5:

         is_supported = False
 
     return is_supported
 
 
         is_supported = False
 
     return is_supported
 
 

-def do_precondition_checks(request, game_meta, raw_players):
-    """Precondition checks for ALL gametypes.
-       These do not require a database connection."""
-    if not has_required_metadata(game_meta):
+def has_minimum_real_players(settings, submission):
+    """
+    Determines if the submission has enough human players to store in the database. The minimum
+    setting comes from the config file under the setting xonstat.minimum_real_players.
+    """
+    try:
+        minimum_required_players = int(settings.get("xonstat.minimum_required_players"))
+    except:
+        minimum_required_players = 2
+
+    return len(submission.humans) >= minimum_required_players
+
+
+def do_precondition_checks(settings, submission):
+    """Precondition checks for ALL gametypes. These do not require a database connection."""
+    if not has_required_metadata(submission):

         msg = "Missing required game metadata"
         log.debug(msg)
         raise pyramid.httpexceptions.HTTPUnprocessableEntity(
         msg = "Missing required game metadata"
         log.debug(msg)
         raise pyramid.httpexceptions.HTTPUnprocessableEntity(


@@ -430,9 +351,7 @@ def do_precondition_checks(request, game_meta, raw_players):
             content_type="text/plain"
         )
 
             content_type="text/plain"
         )
 

-    try:
-        version = int(game_meta['V'])
-    except:
+    if submission.version is None:

         msg = "Invalid or incorrect game metadata provided"
         log.debug(msg)
         raise pyramid.httpexceptions.HTTPUnprocessableEntity(
         msg = "Invalid or incorrect game metadata provided"
         log.debug(msg)
         raise pyramid.httpexceptions.HTTPUnprocessableEntity(


@@ -440,15 +359,15 @@ def do_precondition_checks(request, game_meta, raw_players):
             content_type="text/plain"
         )
 
             content_type="text/plain"
         )
 

-    if not is_supported_gametype(game_meta['G'], version):
-        msg = "Unsupported game type ({})".format(game_meta['G'])
+    if not is_supported_gametype(submission):
+        msg = "Unsupported game type ({})".format(submission.game_type_cd)

         log.debug(msg)
         raise pyramid.httpexceptions.HTTPOk(
             body=msg,
             content_type="text/plain"
         )
 
         log.debug(msg)
         raise pyramid.httpexceptions.HTTPOk(
             body=msg,
             content_type="text/plain"
         )
 

-    if not has_minimum_real_players(request.registry.settings, raw_players):
+    if not has_minimum_real_players(settings, submission):

         msg = "Not enough real players"
         log.debug(msg)
         raise pyramid.httpexceptions.HTTPOk(
         msg = "Not enough real players"
         log.debug(msg)
         raise pyramid.httpexceptions.HTTPOk(


@@ -456,7 +375,7 @@ def do_precondition_checks(request, game_meta, raw_players):
             content_type="text/plain"
         )
 
             content_type="text/plain"
         )
 

-    if is_blank_game(game_meta['G'], raw_players):
+    if is_blank_game(submission):

         msg = "Blank game"
         log.debug(msg)
         raise pyramid.httpexceptions.HTTPOk(
         msg = "Blank game"
         log.debug(msg)
         raise pyramid.httpexceptions.HTTPOk(


@@ -465,74 +384,22 @@ def do_precondition_checks(request, game_meta, raw_players):
         )
 
 
         )
 
 

-def num_real_players(player_events):
-    """
-    Returns the number of real players (those who played
-    and are on the scoreboard).
-    """
-    real_players = 0
-
-    for events in player_events:
-        if is_real_player(events) and played_in_game(events):
-            real_players += 1
-
-    return real_players
-
-
-def has_minimum_real_players(settings, player_events):
-    """
-    Determines if the collection of player events has enough "real" players
-    to store in the database. The minimum setting comes from the config file
-    under the setting xonstat.minimum_real_players.
-    """
-    flg_has_min_real_players = True
-
-    try:
-        minimum_required_players = int(
-                settings['xonstat.minimum_required_players'])
-    except:
-        minimum_required_players = 2
-
-    real_players = num_real_players(player_events)
-
-    if real_players < minimum_required_players:
-        flg_has_min_real_players = False
-
-    return flg_has_min_real_players
-
-
-def has_required_metadata(metadata):
-    """
-    Determines if a give set of metadata has enough data to create a game,
-    server, and map with.
-    """
-    flg_has_req_metadata = True
-
-    if 'G' not in metadata or\
-        'M' not in metadata or\
-        'I' not in metadata or\
-        'S' not in metadata:
-            flg_has_req_metadata = False
-
-    return flg_has_req_metadata
+def get_remote_addr(request):
+    """Get the Xonotic server's IP address"""
+    if 'X-Forwarded-For' in request.headers:
+        return request.headers['X-Forwarded-For']
+    else:
+        return request.remote_addr

 
 
 def should_do_weapon_stats(game_type_cd):
     """True of the game type should record weapon stats. False otherwise."""
 
 
 def should_do_weapon_stats(game_type_cd):
     """True of the game type should record weapon stats. False otherwise."""

-    if game_type_cd in 'cts':
-        return False
-    else:
-        return True
+    return game_type_cd not in {'cts'}

 
 
 def gametype_elo_eligible(game_type_cd):
     """True of the game type should process Elos. False otherwise."""
 
 
 def gametype_elo_eligible(game_type_cd):
     """True of the game type should process Elos. False otherwise."""

-    elo_game_types = ('duel', 'dm', 'ca', 'ctf', 'tdm', 'ka', 'ft')
-
-    if game_type_cd in elo_game_types:
-        return True
-    else:
-        return False
+    return game_type_cd in {'duel', 'dm', 'ca', 'ctf', 'tdm', 'ka', 'ft'}

 
 
 def register_new_nick(session, player, new_nick):
 
 
 def register_new_nick(session, player, new_nick):


@@ -689,54 +556,50 @@ def get_or_create_server(session, name, hashkey, ip_addr, revision, port, impure
     return server
 
 
     return server
 
 

-def get_or_create_map(session=None, name=None):
+def get_or_create_map(session, name):

     """
     Find a map by name or create one if not found. Parameters:
 
     session - SQLAlchemy database session factory
     name - map name of the map to be found or created
     """
     """
     Find a map by name or create one if not found. Parameters:
 
     session - SQLAlchemy database session factory
     name - map name of the map to be found or created
     """

-    try:
-        # find one by the name, if it exists
-        gmap = session.query(Map).filter_by(name=name).one()
-        log.debug("Found map id {0}: {1}".format(gmap.map_id,
-            gmap.name))
-    except NoResultFound, e:
+    maps = session.query(Map).filter_by(name=name).order_by(Map.map_id).all()
+
+    if maps is None or len(maps) == 0:

         gmap = Map(name=name)
         session.add(gmap)
         session.flush()
         gmap = Map(name=name)
         session.add(gmap)
         session.flush()

-        log.debug("Created map id {0}: {1}".format(gmap.map_id,
-            gmap.name))
-    except MultipleResultsFound, e:
-        # multiple found, so use the first one but warn
-        log.debug(e)
-        gmaps = session.query(Map).filter_by(name=name).order_by(
-                Map.map_id).all()
-        gmap = gmaps[0]
-        log.debug("Found map id {0}: {1} but found \
-                multiple".format(gmap.map_id, gmap.name))
+        log.debug("Created map id {}: {}".format(gmap.map_id, gmap.name))
+    elif len(maps) == 1:
+        gmap = maps[0]
+        log.debug("Found map id {}: {}".format(gmap.map_id, gmap.name))
+    else:
+        gmap = maps[0]
+        map_id_list = ", ".join(["{}".format(m.map_id) for m in maps])
+        log.warn("Multiple maps found for {} ({})! Using the first one.".format(name, map_id_list))

 
     return gmap
 
 
 
     return gmap
 
 

-def create_game(session, start_dt, game_type_cd, server_id, map_id,
-        match_id, duration, mod, winner=None):
+def create_game(session, game_type_cd, server_id, map_id, match_id, start_dt, duration, mod,
+                winner=None):

     """
     Creates a game. Parameters:
 
     session - SQLAlchemy database session factory
     """
     Creates a game. Parameters:
 
     session - SQLAlchemy database session factory

-    start_dt - when the game started (datetime object)

     game_type_cd - the game type of the game being played
     game_type_cd - the game type of the game being played

+    mod - mods in use during the game

     server_id - server identifier of the server hosting the game
     map_id - map on which the game was played
     server_id - server identifier of the server hosting the game
     map_id - map on which the game was played

-    winner - the team id of the team that won
+    match_id - a unique match ID given by the server
+    start_dt - when the game started (datetime object)

     duration - how long the game lasted
     duration - how long the game lasted

-    mod - mods in use during the game
+    winner - the team id of the team that won

     """
     seq = Sequence('games_game_id_seq')
     game_id = session.execute(seq)
     """
     seq = Sequence('games_game_id_seq')
     game_id = session.execute(seq)

-    game = Game(game_id=game_id, start_dt=start_dt, game_type_cd=game_type_cd,
-                server_id=server_id, map_id=map_id, winner=winner)
+    game = Game(game_id=game_id, start_dt=start_dt, game_type_cd=game_type_cd, server_id=server_id,
+                map_id=map_id, winner=winner)

     game.match_id = match_id
     game.mod = mod[:64]
 
     game.match_id = match_id
     game.mod = mod[:64]
 


@@ -745,27 +608,25 @@ def create_game(session, start_dt, game_type_cd, server_id, map_id,
     # resolved.
     game.create_dt = start_dt
 
     # resolved.
     game.create_dt = start_dt
 

-    try:
-        game.duration = datetime.timedelta(seconds=int(round(float(duration))))
-    except:
-        pass
+    game.duration = duration

 
     try:
 
     try:

-        session.query(Game).filter(Game.server_id==server_id).\
-                filter(Game.match_id==match_id).one()
+        session.query(Game).filter(Game.server_id == server_id)\
+            .filter(Game.match_id == match_id).one()

 
         log.debug("Error: game with same server and match_id found! Ignoring.")
 
 
         log.debug("Error: game with same server and match_id found! Ignoring.")
 

-        # if a game under the same server and match_id found,
-        # this is a duplicate game and can be ignored
-        raise pyramid.httpexceptions.HTTPOk('OK')
-    except NoResultFound, e:
+        # if a game under the same server_id and match_id exists, this is a duplicate
+        msg = "Duplicate game (pre-existing match_id)"
+        log.debug(msg)
+        raise pyramid.httpexceptions.HTTPOk(body=msg, content_type="text/plain")
+
+    except NoResultFound:

         # server_id/match_id combination not found. game is ok to insert
         session.add(game)
         session.flush()
         # server_id/match_id combination not found. game is ok to insert
         session.add(game)
         session.flush()

-        log.debug("Created game id {0} on server {1}, map {2} at \
-                {3}".format(game.game_id,
-                    server_id, map_id, start_dt))
+        log.debug("Created game id {} on server {}, map {} at {}"
+                  .format(game.game_id, server_id, map_id, start_dt))

 
     return game
 
 
     return game
 


@@ -875,7 +736,7 @@ def create_default_game_stat(session, game_type_cd):
     return pgstat
 
 
     return pgstat
 
 

-def create_game_stat(session, game_meta, game, server, gmap, player, events):
+def create_game_stat(session, game, gmap, player, events):

     """Game stats handler for all game types"""
 
     game_type_cd = game.game_type_cd
     """Game stats handler for all game types"""
 
     game_type_cd = game.game_type_cd


@@ -892,11 +753,6 @@ def create_game_stat(session, game_meta, game, server, gmap, player, events):
     pgstat.rank          = int(events.get('rank', None))
     pgstat.scoreboardpos = int(events.get('scoreboardpos', pgstat.rank))
 
     pgstat.rank          = int(events.get('rank', None))
     pgstat.scoreboardpos = int(events.get('scoreboardpos', pgstat.rank))
 

-    if pgstat.nick != player.nick \
-            and player.player_id > 2 \
-            and pgstat.nick != 'Anonymous Player':
-        register_new_nick(session, player, pgstat.nick)
-

     wins = False
 
     # gametype-specific stuff is handled here. if passed to us, we store it
     wins = False
 
     # gametype-specific stuff is handled here. if passed to us, we store it


@@ -1021,7 +877,7 @@ def create_team_stat(session, game, events):
     return teamstat
 
 
     return teamstat
 
 

-def create_weapon_stats(session, game_meta, game, player, pgstat, events):
+def create_weapon_stats(session, version, game, player, pgstat, events):

     """Weapon stats handler for all game types"""
     pwstats = []
 
     """Weapon stats handler for all game types"""
     pwstats = []
 


@@ -1029,7 +885,6 @@ def create_weapon_stats(session, game_meta, game, player, pgstat, events):
     # To counteract this we divide the data by 2 only for
     # POSTs coming from version 1.
     try:
     # To counteract this we divide the data by 2 only for
     # POSTs coming from version 1.
     try:

-        version = int(game_meta['V'])

         if version == 1:
             is_doubled = True
             log.debug('NOTICE: found a version 1 request, halving the weapon stats...')
         if version == 1:
             is_doubled = True
             log.debug('NOTICE: found a version 1 request, halving the weapon stats...')


@@ -1106,6 +961,87 @@ def get_ranks(session, player_ids, game_type_cd):
     return ranks
 
 
     return ranks
 
 

+def update_player(session, player, events):
+    """
+    Updates a player record using the latest information.
+    :param session: SQLAlchemy session
+    :param player: Player model representing what is in the database right now (before updates)
+    :param events: Dict of player events from the submission
+    :return: player
+    """
+    nick = events.get('n', 'Anonymous Player')[:128]
+    if nick != player.nick and not nick.startswith("Anonymous Player"):
+        register_new_nick(session, player, nick)
+
+    return player
+
+
+def create_player(session, events):
+    """
+    Creates a new player from the list of events.
+    :param session: SQLAlchemy session
+    :param events: Dict of player events from the submission
+    :return: Player
+    """
+    player = Player()
+    session.add(player)
+    session.flush()
+
+    nick = events.get('n', None)
+    if nick:
+        player.nick = nick[:128]
+        player.stripped_nick = strip_colors(qfont_decode(player.nick))
+    else:
+        player.nick = "Anonymous Player #{0}".format(player.player_id)
+        player.stripped_nick = player.nick
+
+    hk = Hashkey(player_id=player.player_id, hashkey=events.get('P', None))
+    session.add(hk)
+
+    return player
+
+
+def get_or_create_players(session, events_by_hashkey):
+    hashkeys = set(events_by_hashkey.keys())
+    players_by_hashkey = {}
+
+    bot = session.query(Player).filter(Player.player_id == 1).one()
+    anon = session.query(Player).filter(Player.player_id == 2).one()
+
+    # fill in the bots and anonymous players
+    for hashkey in events_by_hashkey.keys():
+        if hashkey.startswith("bot#"):
+            players_by_hashkey[hashkey] = bot
+            hashkeys.remove(hashkey)
+        elif hashkey.startswith("player#"):
+            players_by_hashkey[hashkey] = anon
+            hashkeys.remove(hashkey)
+
+    # We are left with the "real" players and can now fetch them by their collective hashkeys.
+    # Those that are returned here are pre-existing players who need to be updated.
+    for p, hk in session.query(Player, Hashkey)\
+            .filter(Player.player_id == Hashkey.player_id)\
+            .filter(Hashkey.hashkey.in_(hashkeys))\
+            .all():
+                log.debug("Found existing player {} with hashkey {}"
+                          .format(p.player_id, hk.hashkey))
+
+                player = update_player(session, p, events_by_hashkey[hk.hashkey])
+                players_by_hashkey[hk.hashkey] = player
+                hashkeys.remove(hk.hashkey)
+
+    # The remainder are the players we haven't seen before, so we need to create them.
+    for hashkey in hashkeys:
+        player = create_player(session, events_by_hashkey[hashkey])
+
+        log.debug("Created player {0} ({2}) with hashkey {1}"
+                  .format(player.player_id, hashkey, player.nick.encode('utf-8')))
+
+        players_by_hashkey[hashkey] = player
+
+    return players_by_hashkey
+
+

 def submit_stats(request):
     """
     Entry handler for POST stats submissions.
 def submit_stats(request):
     """
     Entry handler for POST stats submissions.


@@ -1115,112 +1051,95 @@ def submit_stats(request):
         session = None
 
         log.debug("\n----- BEGIN REQUEST BODY -----\n" + request.body +
         session = None
 
         log.debug("\n----- BEGIN REQUEST BODY -----\n" + request.body +

-                "----- END REQUEST BODY -----\n\n")
+                  "----- END REQUEST BODY -----\n\n")

 
         (idfp, status) = verify_request(request)
 
         (idfp, status) = verify_request(request)

-        (game_meta, raw_players, raw_teams) = parse_stats_submission(request.body)
-        revision = game_meta.get('R', 'unknown')
-        duration = game_meta.get('D', None)
-
-        # only players present at the end of the match are eligible for stats
-        raw_players = filter(played_in_game, raw_players)
-
-        do_precondition_checks(request, game_meta, raw_players)
+        submission = Submission(request.body, request.headers)

 
 

-        # the "duel" gametype is fake
-        if len(raw_players) == 2 \
-            and num_real_players(raw_players) == 2 \
-            and game_meta['G'] == 'dm':
-            game_meta['G'] = 'duel'
+        do_precondition_checks(request.registry.settings, submission)

 
 

-        #----------------------------------------------------------------------
+        #######################################################################

         # Actual setup (inserts/updates) below here
         # Actual setup (inserts/updates) below here

-        #----------------------------------------------------------------------
+        #######################################################################

         session = DBSession()
 
         session = DBSession()
 

-        game_type_cd = game_meta['G']
-

         # All game types create Game, Server, Map, and Player records
         # the same way.
         server = get_or_create_server(
         # All game types create Game, Server, Map, and Player records
         # the same way.
         server = get_or_create_server(

-                session      = session,
-                hashkey      = idfp,
-                name         = game_meta['S'],
-                revision     = revision,
-                ip_addr      = get_remote_addr(request),
-                port         = game_meta.get('U', None),
-                impure_cvars = game_meta.get('C', 0))
-
-        gmap = get_or_create_map(
-                session = session,
-                name    = game_meta['M'])
+            session=session,
+            hashkey=idfp,
+            name=submission.server_name,
+            revision=submission.revision,
+            ip_addr=get_remote_addr(request),
+            port=submission.port_number,
+            impure_cvars=submission.impure_cvar_changes
+        )
+
+        gmap = get_or_create_map(session, submission.map_name)

 
         game = create_game(
 
         game = create_game(

-                session      = session,
-                start_dt     = datetime.datetime.utcnow(),
-                server_id    = server.server_id,
-                game_type_cd = game_type_cd,
-                map_id       = gmap.map_id,
-                match_id     = game_meta['I'],
-                duration     = duration,
-                mod          = game_meta.get('O', None))
-
-        # keep track of the players we've seen
-        player_ids = []
+            session=session,
+            game_type_cd=submission.game_type_cd,
+            mod=submission.mod,
+            server_id=server.server_id,
+            map_id=gmap.map_id,
+            match_id=submission.match_id,
+            start_dt=datetime.datetime.utcnow(),
+            duration=submission.duration
+        )
+
+        events_by_hashkey = {elem["P"]: elem for elem in submission.humans + submission.bots}
+        players_by_hashkey = get_or_create_players(session, events_by_hashkey)
+

         pgstats = []
         pgstats = []

-        hashkeys = {}
-        for events in raw_players:
-            player = get_or_create_player(
-                session = session,
-                hashkey = events['P'],
-                nick    = events.get('n', None))
-
-            pgstat = create_game_stat(session, game_meta, game, server,
-                    gmap, player, events)
+        player_ids = []
+        hashkeys_by_player_id = {}
+        for hashkey, player in players_by_hashkey.items():
+            events = events_by_hashkey[hashkey]
+            pgstat = create_game_stat(session, game, gmap, player, events)

             pgstats.append(pgstat)
 
             if player.player_id > 1:
             pgstats.append(pgstat)
 
             if player.player_id > 1:

-                anticheats = create_anticheats(session, pgstat, game, player, events)
+                create_anticheats(session, pgstat, game, player, events)

 
             if player.player_id > 2:
                 player_ids.append(player.player_id)
 
             if player.player_id > 2:
                 player_ids.append(player.player_id)

-                hashkeys[player.player_id] = events['P']
+                hashkeys_by_player_id[player.player_id] = hashkey

 
 

-            if should_do_weapon_stats(game_type_cd) and player.player_id > 1:
-                pwstats = create_weapon_stats(session, game_meta, game, player,
-                        pgstat, events)
+            if should_do_weapon_stats(submission.game_type_cd) and player.player_id > 1:
+                create_weapon_stats(session, submission.version, game, player, pgstat, events)

 
 

-        # store them on games for easy access
+        # player_ids for human players get stored directly on games for fast indexing

         game.players = player_ids
 
         game.players = player_ids
 

-        for events in raw_teams:
-            try:
-                teamstat = create_team_stat(session, game, events)
-            except Exception as e:
-                raise e
+        for events in submission.teams:
+            create_team_stat(session, game, events)

 
 

-        if server.elo_ind and gametype_elo_eligible(game_type_cd):
+        if server.elo_ind and gametype_elo_eligible(submission.game_type_cd):

             ep = EloProcessor(session, game, pgstats)
             ep.save(session)
             ep = EloProcessor(session, game, pgstats)
             ep.save(session)

+            elos = ep.wip
+        else:
+            elos = {}

 
         session.commit()
         log.debug('Success! Stats recorded.')
 
         # ranks are fetched after we've done the "real" processing
 
         session.commit()
         log.debug('Success! Stats recorded.')
 
         # ranks are fetched after we've done the "real" processing

-        ranks = get_ranks(session, player_ids, game_type_cd)
+        ranks = get_ranks(session, player_ids, submission.game_type_cd)

 
         # plain text response
         request.response.content_type = 'text/plain'
 
         return {
 
         # plain text response
         request.response.content_type = 'text/plain'
 
         return {

-                "now"        : calendar.timegm(datetime.datetime.utcnow().timetuple()),
-                "server"     : server,
-                "game"       : game,
-                "gmap"       : gmap,
-                "player_ids" : player_ids,
-                "hashkeys"   : hashkeys,
-                "elos"       : ep.wip,
-                "ranks"      : ranks,
+                "now": calendar.timegm(datetime.datetime.utcnow().timetuple()),
+                "server": server,
+                "game": game,
+                "gmap": gmap,
+                "player_ids": player_ids,
+                "hashkeys": hashkeys_by_player_id,
+                "elos": elos,
+                "ranks": ranks,

         }
 
     except Exception as e:
         }
 
     except Exception as e: