X-Git-Url: http://de.git.xonotic.org/?p=xonotic%2Fxonstat.git;a=blobdiff_plain;f=xonstat%2Fviews%2Fsubmission.py;h=8a00bc8a6d60c0d8a72b2dcd56f74e4f0748c159;hp=75c0f017e9b83538ceeb8c3b2a034b63e89fd4ec;hb=5e1c717036fdc0523794cc90aaefc5c1d6bd4ee0;hpb=a4ec7db887faa82ee42695b3045a8f6501edef05 diff --git a/xonstat/views/submission.py b/xonstat/views/submission.py index 75c0f01..8a00bc8 100644 --- a/xonstat/views/submission.py +++ b/xonstat/views/submission.py @@ -1,10 +1,10 @@ import calendar +import collections import datetime import logging import re import pyramid.httpexceptions -import sqlalchemy.sql.expression as expr from sqlalchemy import Sequence from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound from xonstat.elo import EloProcessor @@ -16,6 +16,259 @@ from xonstat.util import strip_colors, qfont_decode, verify_request, weapon_map log = logging.getLogger(__name__) +def is_real_player(events): + """ + Determines if a given set of events correspond with a non-bot + """ + return not events['P'].startswith('bot') + + +def played_in_game(events): + """ + Determines if a given set of player events correspond with a player who + played in the game (matches 1 and scoreboardvalid 1) + """ + return 'matches' in events and 'scoreboardvalid' in events + + +class Submission(object): + """Parses an incoming POST request for stats submissions.""" + + def __init__(self, body, headers): + # a copy of the HTTP headers + self.headers = headers + + # a copy of the HTTP POST body + self.body = body + + # the submission code version (from the server) + self.version = None + + # the revision string of the server + self.revision = None + + # the game type played + self.game_type_cd = None + + # the active game mod + self.mod = None + + # the name of the map played + self.map_name = None + + # unique identifier (string) for a match on a given server + self.match_id = None + + # the name of the server + self.server_name = None + + # the number of cvars that were changed to be different than default + self.impure_cvar_changes = None + + # the port number the game server is listening on + self.port_number = None + + # how long the game lasted + self.duration = None + + # which ladder is being used, if any + self.ladder = None + + # players involved in the match (humans, bots, and spectators) + self.players = [] + + # raw team events + self.teams = [] + + # the parsing deque (we use this to allow peeking) + self.q = collections.deque(self.body.split("\n")) + + ############################################################################################ + # Below this point are fields useful in determining if the submission is valid or + # performance optimizations that save us from looping over the events over and over again. + ############################################################################################ + + # humans who played in the match + self.humans = [] + + # bots who played in the match + self.bots = [] + + # distinct weapons that we have seen fired + self.weapons = set() + + # has a human player fired a shot? + self.human_fired_weapon = False + + # does any human have a non-zero score? + self.human_nonzero_score = False + + # does any human have a fastest cap? + self.human_fastest = False + + def next_item(self): + """Returns the next key:value pair off the queue.""" + try: + items = self.q.popleft().strip().split(' ', 1) + if len(items) == 1: + # Some keys won't have values, like 'L' records where the server isn't actually + # participating in any ladders. These can be safely ignored. + return None, None + else: + return items + except: + return None, None + + def add_weapon_fired(self, sub_key): + """Adds a weapon to the set of weapons fired during the match (a set).""" + self.weapons.add(sub_key.split("-")[1]) + + @staticmethod + def is_human_player(player): + """ + Determines if a given set of events correspond with a non-bot + """ + return not player['P'].startswith('bot') + + @staticmethod + def played_in_game(player): + """ + Determines if a given set of player events correspond with a player who + played in the game (matches 1 and scoreboardvalid 1) + """ + return 'matches' in player and 'scoreboardvalid' in player + + def parse_player(self, key, pid): + """Construct a player events listing from the submission.""" + + # all of the keys related to player records + player_keys = ['i', 'n', 't', 'e'] + + player = {key: pid} + + player_fired_weapon = False + player_nonzero_score = False + player_fastest = False + + # Consume all following 'i' 'n' 't' 'e' records + while len(self.q) > 0: + (key, value) = self.next_item() + if key is None and value is None: + continue + elif key == 'e': + (sub_key, sub_value) = value.split(' ', 1) + player[sub_key] = sub_value + + if sub_key.endswith("cnt-fired"): + player_fired_weapon = True + self.add_weapon_fired(sub_key) + elif sub_key == 'scoreboard-score' and int(sub_value) != 0: + player_nonzero_score = True + elif sub_key == 'scoreboard-fastest': + player_fastest = True + elif key == 'n': + player[key] = unicode(value, 'utf-8') + elif key in player_keys: + player[key] = value + else: + # something we didn't expect - put it back on the deque + self.q.appendleft("{} {}".format(key, value)) + break + + played = self.played_in_game(player) + human = self.is_human_player(player) + + if played and human: + self.humans.append(player) + + if player_fired_weapon: + self.human_fired_weapon = True + + if player_nonzero_score: + self.human_nonzero_score = True + + if player_fastest: + self.human_fastest = True + + elif played and not human: + self.bots.append(player) + else: + self.players.append(player) + + def parse_team(self, key, tid): + """Construct a team events listing from the submission.""" + team = {key: tid} + + # Consume all following 'e' records + while len(self.q) > 0 and self.q[0].startswith('e'): + (_, value) = self.next_item() + (sub_key, sub_value) = value.split(' ', 1) + team[sub_key] = sub_value + + self.teams.append(team) + + def parse(self): + """Parses the request body into instance variables.""" + while len(self.q) > 0: + (key, value) = self.next_item() + if key is None and value is None: + continue + elif key == 'V': + self.version = value + elif key == 'R': + self.revision = value + elif key == 'G': + self.game_type_cd = value + elif key == 'O': + self.mod = value + elif key == 'M': + self.map_name = value + elif key == 'I': + self.match_id = value + elif key == 'S': + self.server_name = unicode(value, 'utf-8') + elif key == 'C': + self.impure_cvar_changes = int(value) + elif key == 'U': + self.port_number = int(value) + elif key == 'D': + self.duration = datetime.timedelta(seconds=int(round(float(value)))) + elif key == 'L': + self.ladder = value + elif key == 'Q': + self.parse_team(key, value) + elif key == 'P': + self.parse_player(key, value) + else: + raise Exception("Invalid submission") + + return self + + +def elo_submission_category(submission): + """Determines the Elo category purely by what is in the submission data.""" + mod = submission.meta.get("O", "None") + + vanilla_allowed_weapons = {"shotgun", "devastator", "blaster", "mortar", "vortex", "electro", + "arc", "hagar", "crylink", "machinegun"} + insta_allowed_weapons = {"vaporizer", "blaster"} + overkill_allowed_weapons = {"hmg", "vortex", "shotgun", "blaster", "machinegun", "rpc"} + + if mod == "Xonotic": + if len(submission.weapons - vanilla_allowed_weapons) == 0: + return "vanilla" + elif mod == "InstaGib": + if len(submission.weapons - insta_allowed_weapons) == 0: + return "insta" + elif mod == "Overkill": + if len(submission.weapons - overkill_allowed_weapons) == 0: + return "overkill" + else: + return "general" + + return "general" + + def parse_stats_submission(body): """ Parses the POST request body for a stats submission @@ -150,6 +403,7 @@ def is_supported_gametype(gametype, version): 'cts', 'dm', 'dom', + 'duel', 'ft', 'freezetag', 'ka', 'keepaway', 'kh', @@ -176,47 +430,46 @@ def do_precondition_checks(request, game_meta, raw_players): """Precondition checks for ALL gametypes. These do not require a database connection.""" if not has_required_metadata(game_meta): - log.debug("ERROR: Required game meta missing") - raise pyramid.httpexceptions.HTTPUnprocessableEntity("Missing game meta") + msg = "Missing required game metadata" + log.debug(msg) + raise pyramid.httpexceptions.HTTPUnprocessableEntity( + body=msg, + content_type="text/plain" + ) try: version = int(game_meta['V']) except: - log.debug("ERROR: Required game meta invalid") - raise pyramid.httpexceptions.HTTPUnprocessableEntity("Invalid game meta") + msg = "Invalid or incorrect game metadata provided" + log.debug(msg) + raise pyramid.httpexceptions.HTTPUnprocessableEntity( + body=msg, + content_type="text/plain" + ) if not is_supported_gametype(game_meta['G'], version): - log.debug("ERROR: Unsupported gametype") - raise pyramid.httpexceptions.HTTPOk("OK") + msg = "Unsupported game type ({})".format(game_meta['G']) + log.debug(msg) + raise pyramid.httpexceptions.HTTPOk( + body=msg, + content_type="text/plain" + ) if not has_minimum_real_players(request.registry.settings, raw_players): - log.debug("ERROR: Not enough real players") - raise pyramid.httpexceptions.HTTPOk("OK") + msg = "Not enough real players" + log.debug(msg) + raise pyramid.httpexceptions.HTTPOk( + body=msg, + content_type="text/plain" + ) if is_blank_game(game_meta['G'], raw_players): - log.debug("ERROR: Blank game") - raise pyramid.httpexceptions.HTTPOk("OK") - - -def is_real_player(events): - """ - Determines if a given set of events correspond with a non-bot - """ - if not events['P'].startswith('bot'): - return True - else: - return False - - -def played_in_game(events): - """ - Determines if a given set of player events correspond with a player who - played in the game (matches 1 and scoreboardvalid 1) - """ - if 'matches' in events and 'scoreboardvalid' in events: - return True - else: - return False + msg = "Blank game" + log.debug(msg) + raise pyramid.httpexceptions.HTTPOk( + body=msg, + content_type="text/plain" + ) def num_real_players(player_events): @@ -350,86 +603,94 @@ def update_fastest_cap(session, player_id, game_id, map_id, captime, mod): session.flush() -def get_or_create_server(session, name, hashkey, ip_addr, revision, port, - impure_cvars): +def update_server(server, name, hashkey, ip_addr, port, revision, impure_cvars): """ - Find a server by name or create one if not found. Parameters: - - session - SQLAlchemy database session factory - name - server name of the server to be found or created - hashkey - server hashkey - ip_addr - the IP address of the server - revision - the xonotic revision number - port - the port number of the server - impure_cvars - the number of impure cvar changes + Updates the server in the given DB session, if needed. + + :param server: The found server instance. + :param name: The incoming server name. + :param hashkey: The incoming server hashkey. + :param ip_addr: The incoming server IP address. + :param port: The incoming server port. + :param revision: The incoming server revision. + :param impure_cvars: The incoming number of impure server cvars. + :return: bool """ - server = None - + # ensure the two int attributes are actually ints try: port = int(port) except: port = None - try: + try: impure_cvars = int(impure_cvars) except: impure_cvars = 0 - # finding by hashkey is preferred, but if not we will fall - # back to using name only, which can result in dupes - if hashkey is not None: - servers = session.query(Server).\ - filter_by(hashkey=hashkey).\ - order_by(expr.desc(Server.create_dt)).limit(1).all() - - if len(servers) > 0: - server = servers[0] - log.debug("Found existing server {0} by hashkey ({1})".format( - server.server_id, server.hashkey)) - else: - servers = session.query(Server).\ - filter_by(name=name).\ - order_by(expr.desc(Server.create_dt)).limit(1).all() + updated = False + if name and server.name != name: + server.name = name + updated = True + if hashkey and server.hashkey != hashkey: + server.hashkey = hashkey + updated = True + if ip_addr and server.ip_addr != ip_addr: + server.ip_addr = ip_addr + updated = True + if port and server.port != port: + server.port = port + updated = True + if revision and server.revision != revision: + server.revision = revision + updated = True + if impure_cvars and server.impure_cvars != impure_cvars: + server.impure_cvars = impure_cvars + server.pure_ind = True if impure_cvars == 0 else False + updated = True - if len(servers) > 0: - server = servers[0] - log.debug("Found existing server {0} by name".format(server.server_id)) + return updated - # still haven't found a server by hashkey or name, so we need to create one - if server is None: - server = Server(name=name, hashkey=hashkey) - session.add(server) - session.flush() - log.debug("Created server {0} with hashkey {1}".format( - server.server_id, server.hashkey)) - # detect changed fields - if server.name != name: - server.name = name - session.add(server) +def get_or_create_server(session, name, hashkey, ip_addr, revision, port, impure_cvars): + """ + Find a server by name or create one if not found. Parameters: - if server.hashkey != hashkey: - server.hashkey = hashkey - session.add(server) + session - SQLAlchemy database session factory + name - server name of the server to be found or created + hashkey - server hashkey + ip_addr - the IP address of the server + revision - the xonotic revision number + port - the port number of the server + impure_cvars - the number of impure cvar changes + """ + servers_q = DBSession.query(Server).filter(Server.active_ind) - if server.ip_addr != ip_addr: - server.ip_addr = ip_addr - session.add(server) + if hashkey: + # if the hashkey is provided, we'll use that + servers_q = servers_q.filter((Server.name == name) or (Server.hashkey == hashkey)) + else: + # otherwise, it is just by name + servers_q = servers_q.filter(Server.name == name) - if server.port != port: - server.port = port - session.add(server) + # order by the hashkey, which means any hashkey match will appear first if there are multiple + servers = servers_q.order_by(Server.hashkey, Server.create_dt).all() - if server.revision != revision: - server.revision = revision + if len(servers) == 0: + server = Server(name=name, hashkey=hashkey) session.add(server) + session.flush() + log.debug("Created server {} with hashkey {}.".format(server.server_id, server.hashkey)) + else: + server = servers[0] + if len(servers) == 1: + log.info("Found existing server {}.".format(server.server_id)) - if server.impure_cvars != impure_cvars: - server.impure_cvars = impure_cvars - if impure_cvars > 0: - server.pure_ind = False - else: - server.pure_ind = True + elif len(servers) > 1: + server_id_list = ", ".join(["{}".format(s.server_id) for s in servers]) + log.warn("Multiple servers found ({})! Using the first one ({})." + .format(server_id_list, server.server_id)) + + if update_server(server, name, hashkey, ip_addr, port, revision, impure_cvars): session.add(server) return server