Fix for scoreboard-scores being float values.

[xonotic/xonstat.git] / xonstat / views / submission.py

diff --git a/xonstat/views/submission.py b/xonstat/views/submission.py
index 670807b8ad0c403a14d167189eac51e0eae1b1e7..b4059e5ad06c2bbe3863587c7b3499ed0f5521b1 100644 (file)
--- a/xonstat/views/submission.py
+++ b/xonstat/views/submission.py
@@ -6,7 +6,7 @@ import re
 
 import pyramid.httpexceptions
 from sqlalchemy import Sequence
 
 import pyramid.httpexceptions
 from sqlalchemy import Sequence

-from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound
+from sqlalchemy.orm.exc import NoResultFound

 from xonstat.elo import EloProcessor
 from xonstat.models import DBSession, Server, Map, Game, PlayerGameStat, PlayerWeaponStat
 from xonstat.models import PlayerRank, PlayerCaptime
 from xonstat.elo import EloProcessor
 from xonstat.models import DBSession, Server, Map, Game, PlayerGameStat, PlayerWeaponStat
 from xonstat.models import PlayerRank, PlayerCaptime


@@ -91,6 +91,8 @@ class Submission(object):
         # does any human have a fastest cap?
         self.human_fastest = False
 
         # does any human have a fastest cap?
         self.human_fastest = False
 

+        self.parse()
+

     def next_item(self):
         """Returns the next key:value pair off the queue."""
         try:
     def next_item(self):
         """Returns the next key:value pair off the queue."""
         try:


@@ -127,7 +129,7 @@ class Submission(object):
         """Construct a player events listing from the submission."""
 
         # all of the keys related to player records
         """Construct a player events listing from the submission."""
 
         # all of the keys related to player records

-        player_keys = ['i', 'n', 't', 'e']
+        player_keys = ['i', 'n', 't', 'r', 'e']

 
         player = {key: pid}
 
 
         player = {key: pid}
 


@@ -147,7 +149,7 @@ class Submission(object):
                 if sub_key.endswith("cnt-fired"):
                     player_fired_weapon = True
                     self.add_weapon_fired(sub_key)
                 if sub_key.endswith("cnt-fired"):
                     player_fired_weapon = True
                     self.add_weapon_fired(sub_key)

-                elif sub_key == 'scoreboard-score' and int(sub_value) != 0:
+                elif sub_key == 'scoreboard-score' and int(round(float(sub_value))) != 0:

                     player_nonzero_score = True
                 elif sub_key == 'scoreboard-fastest':
                     player_fastest = True
                     player_nonzero_score = True
                 elif sub_key == 'scoreboard-fastest':
                     player_fastest = True


@@ -338,7 +340,7 @@ def has_minimum_real_players(settings, submission):
     except:
         minimum_required_players = 2
 
     except:
         minimum_required_players = 2
 

-    return len(submission.human_players) >= minimum_required_players
+    return len(submission.humans) >= minimum_required_players

 
 
 def do_precondition_checks(settings, submission):
 
 
 def do_precondition_checks(settings, submission):


@@ -556,54 +558,50 @@ def get_or_create_server(session, name, hashkey, ip_addr, revision, port, impure
     return server
 
 
     return server
 
 

-def get_or_create_map(session=None, name=None):
+def get_or_create_map(session, name):

     """
     Find a map by name or create one if not found. Parameters:
 
     session - SQLAlchemy database session factory
     name - map name of the map to be found or created
     """
     """
     Find a map by name or create one if not found. Parameters:
 
     session - SQLAlchemy database session factory
     name - map name of the map to be found or created
     """

-    try:
-        # find one by the name, if it exists
-        gmap = session.query(Map).filter_by(name=name).one()
-        log.debug("Found map id {0}: {1}".format(gmap.map_id,
-            gmap.name))
-    except NoResultFound, e:
+    maps = session.query(Map).filter_by(name=name).order_by(Map.map_id).all()
+
+    if maps is None or len(maps) == 0:

         gmap = Map(name=name)
         session.add(gmap)
         session.flush()
         gmap = Map(name=name)
         session.add(gmap)
         session.flush()

-        log.debug("Created map id {0}: {1}".format(gmap.map_id,
-            gmap.name))
-    except MultipleResultsFound, e:
-        # multiple found, so use the first one but warn
-        log.debug(e)
-        gmaps = session.query(Map).filter_by(name=name).order_by(
-                Map.map_id).all()
-        gmap = gmaps[0]
-        log.debug("Found map id {0}: {1} but found \
-                multiple".format(gmap.map_id, gmap.name))
+        log.debug("Created map id {}: {}".format(gmap.map_id, gmap.name))
+    elif len(maps) == 1:
+        gmap = maps[0]
+        log.debug("Found map id {}: {}".format(gmap.map_id, gmap.name))
+    else:
+        gmap = maps[0]
+        map_id_list = ", ".join(["{}".format(m.map_id) for m in maps])
+        log.warn("Multiple maps found for {} ({})! Using the first one.".format(name, map_id_list))

 
     return gmap
 
 
 
     return gmap
 
 

-def create_game(session, start_dt, game_type_cd, server_id, map_id,
-        match_id, duration, mod, winner=None):
+def create_game(session, game_type_cd, server_id, map_id, match_id, start_dt, duration, mod,
+                winner=None):

     """
     Creates a game. Parameters:
 
     session - SQLAlchemy database session factory
     """
     Creates a game. Parameters:
 
     session - SQLAlchemy database session factory

-    start_dt - when the game started (datetime object)

     game_type_cd - the game type of the game being played
     game_type_cd - the game type of the game being played

+    mod - mods in use during the game

     server_id - server identifier of the server hosting the game
     map_id - map on which the game was played
     server_id - server identifier of the server hosting the game
     map_id - map on which the game was played

-    winner - the team id of the team that won
+    match_id - a unique match ID given by the server
+    start_dt - when the game started (datetime object)

     duration - how long the game lasted
     duration - how long the game lasted

-    mod - mods in use during the game
+    winner - the team id of the team that won

     """
     seq = Sequence('games_game_id_seq')
     game_id = session.execute(seq)
     """
     seq = Sequence('games_game_id_seq')
     game_id = session.execute(seq)

-    game = Game(game_id=game_id, start_dt=start_dt, game_type_cd=game_type_cd,
-                server_id=server_id, map_id=map_id, winner=winner)
+    game = Game(game_id=game_id, start_dt=start_dt, game_type_cd=game_type_cd, server_id=server_id,
+                map_id=map_id, winner=winner)

     game.match_id = match_id
     game.mod = mod[:64]
 
     game.match_id = match_id
     game.mod = mod[:64]
 


@@ -612,27 +610,25 @@ def create_game(session, start_dt, game_type_cd, server_id, map_id,
     # resolved.
     game.create_dt = start_dt
 
     # resolved.
     game.create_dt = start_dt
 

-    try:
-        game.duration = datetime.timedelta(seconds=int(round(float(duration))))
-    except:
-        pass
+    game.duration = duration

 
     try:
 
     try:

-        session.query(Game).filter(Game.server_id==server_id).\
-                filter(Game.match_id==match_id).one()
+        session.query(Game).filter(Game.server_id == server_id)\
+            .filter(Game.match_id == match_id).one()

 
         log.debug("Error: game with same server and match_id found! Ignoring.")
 
 
         log.debug("Error: game with same server and match_id found! Ignoring.")
 

-        # if a game under the same server and match_id found,
-        # this is a duplicate game and can be ignored
-        raise pyramid.httpexceptions.HTTPOk('OK')
-    except NoResultFound, e:
+        # if a game under the same server_id and match_id exists, this is a duplicate
+        msg = "Duplicate game (pre-existing match_id)"
+        log.debug(msg)
+        raise pyramid.httpexceptions.HTTPOk(body=msg, content_type="text/plain")
+
+    except NoResultFound:

         # server_id/match_id combination not found. game is ok to insert
         session.add(game)
         session.flush()
         # server_id/match_id combination not found. game is ok to insert
         session.add(game)
         session.flush()

-        log.debug("Created game id {0} on server {1}, map {2} at \
-                {3}".format(game.game_id,
-                    server_id, map_id, start_dt))
+        log.debug("Created game id {} on server {}, map {} at {}"
+                  .format(game.game_id, server_id, map_id, start_dt))

 
     return game
 
 
     return game
 


@@ -759,11 +755,6 @@ def create_game_stat(session, game, gmap, player, events):
     pgstat.rank          = int(events.get('rank', None))
     pgstat.scoreboardpos = int(events.get('scoreboardpos', pgstat.rank))
 
     pgstat.rank          = int(events.get('rank', None))
     pgstat.scoreboardpos = int(events.get('scoreboardpos', pgstat.rank))
 

-    if pgstat.nick != player.nick \
-            and player.player_id > 2 \
-            and pgstat.nick != 'Anonymous Player':
-        register_new_nick(session, player, pgstat.nick)
-

     wins = False
 
     # gametype-specific stuff is handled here. if passed to us, we store it
     wins = False
 
     # gametype-specific stuff is handled here. if passed to us, we store it


@@ -972,25 +963,114 @@ def get_ranks(session, player_ids, game_type_cd):
     return ranks
 
 
     return ranks
 
 

+def update_player(session, player, events):
+    """
+    Updates a player record using the latest information.
+    :param session: SQLAlchemy session
+    :param player: Player model representing what is in the database right now (before updates)
+    :param events: Dict of player events from the submission
+    :return: player
+    """
+    nick = events.get('n', 'Anonymous Player')[:128]
+    if nick != player.nick and not nick.startswith("Anonymous Player"):
+        register_new_nick(session, player, nick)
+
+    return player
+
+
+def create_player(session, events):
+    """
+    Creates a new player from the list of events.
+    :param session: SQLAlchemy session
+    :param events: Dict of player events from the submission
+    :return: Player
+    """
+    player = Player()
+    session.add(player)
+    session.flush()
+
+    nick = events.get('n', None)
+    if nick:
+        player.nick = nick[:128]
+        player.stripped_nick = strip_colors(qfont_decode(player.nick))
+    else:
+        player.nick = "Anonymous Player #{0}".format(player.player_id)
+        player.stripped_nick = player.nick
+
+    hk = Hashkey(player_id=player.player_id, hashkey=events.get('P', None))
+    session.add(hk)
+
+    return player
+
+
+def get_or_create_players(session, events_by_hashkey):
+    hashkeys = set(events_by_hashkey.keys())
+    players_by_hashkey = {}
+
+    bot = session.query(Player).filter(Player.player_id == 1).one()
+    anon = session.query(Player).filter(Player.player_id == 2).one()
+
+    # fill in the bots and anonymous players
+    for hashkey in events_by_hashkey.keys():
+        if hashkey.startswith("bot#"):
+            players_by_hashkey[hashkey] = bot
+            hashkeys.remove(hashkey)
+        elif hashkey.startswith("player#"):
+            players_by_hashkey[hashkey] = anon
+            hashkeys.remove(hashkey)
+
+    # We are left with the "real" players and can now fetch them by their collective hashkeys.
+    # Those that are returned here are pre-existing players who need to be updated.
+    for p, hk in session.query(Player, Hashkey)\
+            .filter(Player.player_id == Hashkey.player_id)\
+            .filter(Hashkey.hashkey.in_(hashkeys))\
+            .all():
+                log.debug("Found existing player {} with hashkey {}"
+                          .format(p.player_id, hk.hashkey))
+
+                player = update_player(session, p, events_by_hashkey[hk.hashkey])
+                players_by_hashkey[hk.hashkey] = player
+                hashkeys.remove(hk.hashkey)
+
+    # The remainder are the players we haven't seen before, so we need to create them.
+    for hashkey in hashkeys:
+        player = create_player(session, events_by_hashkey[hashkey])
+
+        log.debug("Created player {0} ({2}) with hashkey {1}"
+                  .format(player.player_id, hashkey, player.nick.encode('utf-8')))
+
+        players_by_hashkey[hashkey] = player
+
+    return players_by_hashkey
+
+

 def submit_stats(request):
     """
     Entry handler for POST stats submissions.
     """
 def submit_stats(request):
     """
     Entry handler for POST stats submissions.
     """

-    try:
-        # placeholder for the actual session
-        session = None
+    # placeholder for the actual session
+    session = None

 
 

+    try:

         log.debug("\n----- BEGIN REQUEST BODY -----\n" + request.body +
         log.debug("\n----- BEGIN REQUEST BODY -----\n" + request.body +

-                "----- END REQUEST BODY -----\n\n")
+                  "----- END REQUEST BODY -----\n\n")

 
         (idfp, status) = verify_request(request)
 
         (idfp, status) = verify_request(request)

-        submission = Submission(request.body, request.headers)
+        try:
+            submission = Submission(request.body, request.headers)
+        except:
+            msg = "Invalid submission"
+            log.debug(msg)
+            raise pyramid.httpexceptions.HTTPUnprocessableEntity(
+                body=msg,
+                content_type="text/plain"
+            )

 
         do_precondition_checks(request.registry.settings, submission)
 
 
         do_precondition_checks(request.registry.settings, submission)
 

-        #----------------------------------------------------------------------
+        #######################################################################

         # Actual setup (inserts/updates) below here
         # Actual setup (inserts/updates) below here

-        #----------------------------------------------------------------------
+        #######################################################################

         session = DBSession()
 
         # All game types create Game, Server, Map, and Player records
         session = DBSession()
 
         # All game types create Game, Server, Map, and Player records


@@ -1009,46 +1089,53 @@ def submit_stats(request):
 
         game = create_game(
             session=session,
 
         game = create_game(
             session=session,

-            start_dt=datetime.datetime.utcnow(),
-            server_id=server.server_id,

             game_type_cd=submission.game_type_cd,
             game_type_cd=submission.game_type_cd,

+            mod=submission.mod,
+            server_id=server.server_id,

             map_id=gmap.map_id,
             match_id=submission.match_id,
             map_id=gmap.map_id,
             match_id=submission.match_id,

-            duration=submission.duration,
-            mod=submission.mod
+            start_dt=datetime.datetime.utcnow(),
+            duration=submission.duration

         )
 
         )
 

-        # keep track of the players we've seen
-        player_ids = []
+        events_by_hashkey = {elem["P"]: elem for elem in submission.humans + submission.bots}
+        players_by_hashkey = get_or_create_players(session, events_by_hashkey)
+

         pgstats = []
         pgstats = []

-        hashkeys = {}
-        for events in submission.humans + submission.bots:
-            player = get_or_create_player(session, events['P'], events.get('n', None))
+        elo_pgstats = []
+        player_ids = []
+        hashkeys_by_player_id = {}
+        for hashkey, player in players_by_hashkey.items():
+            events = events_by_hashkey[hashkey]

             pgstat = create_game_stat(session, game, gmap, player, events)
             pgstats.append(pgstat)
 
             pgstat = create_game_stat(session, game, gmap, player, events)
             pgstats.append(pgstat)
 

+            # player ranking opt-out
+            if 'r' in events and events['r'] != '0':
+                elo_pgstats.append(pgstat)
+

             if player.player_id > 1:
                 create_anticheats(session, pgstat, game, player, events)
 
             if player.player_id > 2:
                 player_ids.append(player.player_id)
             if player.player_id > 1:
                 create_anticheats(session, pgstat, game, player, events)
 
             if player.player_id > 2:
                 player_ids.append(player.player_id)

-                hashkeys[player.player_id] = events['P']
+                hashkeys_by_player_id[player.player_id] = hashkey

 
             if should_do_weapon_stats(submission.game_type_cd) and player.player_id > 1:
                 create_weapon_stats(session, submission.version, game, player, pgstat, events)
 
 
             if should_do_weapon_stats(submission.game_type_cd) and player.player_id > 1:
                 create_weapon_stats(session, submission.version, game, player, pgstat, events)
 

-        # store them on games for easy access
+        # player_ids for human players get stored directly on games for fast indexing

         game.players = player_ids
 
         for events in submission.teams:
         game.players = player_ids
 
         for events in submission.teams:

-            try:
-                create_team_stat(session, game, events)
-            except Exception as e:
-                raise e
+            create_team_stat(session, game, events)

 
         if server.elo_ind and gametype_elo_eligible(submission.game_type_cd):
 
         if server.elo_ind and gametype_elo_eligible(submission.game_type_cd):

-            ep = EloProcessor(session, game, pgstats)
+            ep = EloProcessor(session, game, elo_pgstats)

             ep.save(session)
             ep.save(session)

+            elos = ep.wip
+        else:
+            elos = {}

 
         session.commit()
         log.debug('Success! Stats recorded.')
 
         session.commit()
         log.debug('Success! Stats recorded.')


@@ -1065,8 +1152,8 @@ def submit_stats(request):
                 "game": game,
                 "gmap": gmap,
                 "player_ids": player_ids,
                 "game": game,
                 "gmap": gmap,
                 "player_ids": player_ids,

-                "hashkeys": hashkeys,
-                "elos": ep.wip,
+                "hashkeys": hashkeys_by_player_id,
+                "elos": elos,

                 "ranks": ranks,
         }
 
                 "ranks": ranks,
         }