X-Git-Url: http://de.git.xonotic.org/?a=blobdiff_plain;f=xonstat%2Fviews%2Fsubmission.py;h=2e89c15d5379d051232f889c623566a9ff85edce;hb=0ce1a9b98d4f5f94d70499d913e61e9f671f5662;hp=0d3a2bbde04e13ec16b40d45e85b201f18efe2c3;hpb=04f5adb0164041e4ccdf878b611387b0fbad46a1;p=xonotic%2Fxonstat.git diff --git a/xonstat/views/submission.py b/xonstat/views/submission.py index 0d3a2bb..2e89c15 100644 --- a/xonstat/views/submission.py +++ b/xonstat/views/submission.py @@ -16,27 +16,6 @@ from xonstat.util import strip_colors, qfont_decode, verify_request, weapon_map log = logging.getLogger(__name__) -def is_real_player(events): - """ - Determines if a given set of events correspond with a non-bot - """ - if not events['P'].startswith('bot'): - return True - else: - return False - - -def played_in_game(events): - """ - Determines if a given set of player events correspond with a player who - played in the game (matches 1 and scoreboardvalid 1) - """ - if 'matches' in events and 'scoreboardvalid' in events: - return True - else: - return False - - class Submission(object): """Parses an incoming POST request for stats submissions.""" @@ -47,29 +26,70 @@ class Submission(object): # a copy of the HTTP POST body self.body = body - # game metadata - self.meta = {} + # the submission code version (from the server) + self.version = None + + # the revision string of the server + self.revision = None + + # the game type played + self.game_type_cd = None + + # the active game mod + self.mod = None + + # the name of the map played + self.map_name = None - # humans and bots in the match (including spectators) + # unique identifier (string) for a match on a given server + self.match_id = None + + # the name of the server + self.server_name = None + + # the number of cvars that were changed to be different than default + self.impure_cvar_changes = None + + # the port number the game server is listening on + self.port_number = None + + # how long the game lasted + self.duration = None + + # which ladder is being used, if any + self.ladder = None + + # players involved in the match (humans, bots, and spectators) self.players = [] + # raw team events + self.teams = [] + + # the parsing deque (we use this to allow peeking) + self.q = collections.deque(self.body.split("\n")) + + ############################################################################################ + # Below this point are fields useful in determining if the submission is valid or + # performance optimizations that save us from looping over the events over and over again. + ############################################################################################ + # humans who played in the match self.humans = [] # bots who played in the match self.bots = [] - # raw team events - self.teams = [] - # distinct weapons that we have seen fired self.weapons = set() # has a human player fired a shot? self.human_fired_weapon = False - # the parsing deque (we use this to allow peeking) - self.q = collections.deque(self.body.split("\n")) + # does any human have a non-zero score? + self.human_nonzero_score = False + + # does any human have a fastest cap? + self.human_fastest = False def next_item(self): """Returns the next key:value pair off the queue.""" @@ -84,11 +104,24 @@ class Submission(object): except: return None, None - def check_for_new_weapon_fired(self, sub_key): - """Checks if a given weapon fired event is a new one for the match.""" - weapon = sub_key.split("-")[1] - if weapon not in self.weapons: - self.weapons.add(weapon) + def add_weapon_fired(self, sub_key): + """Adds a weapon to the set of weapons fired during the match (a set).""" + self.weapons.add(sub_key.split("-")[1]) + + @staticmethod + def is_human_player(player): + """ + Determines if a given set of events correspond with a non-bot + """ + return not player['P'].startswith('bot') + + @staticmethod + def played_in_game(player): + """ + Determines if a given set of player events correspond with a player who + played in the game (matches 1 and scoreboardvalid 1) + """ + return 'matches' in player and 'scoreboardvalid' in player def parse_player(self, key, pid): """Construct a player events listing from the submission.""" @@ -99,6 +132,8 @@ class Submission(object): player = {key: pid} player_fired_weapon = False + player_nonzero_score = False + player_fastest = False # Consume all following 'i' 'n' 't' 'e' records while len(self.q) > 0: @@ -111,7 +146,11 @@ class Submission(object): if sub_key.endswith("cnt-fired"): player_fired_weapon = True - self.check_for_new_weapon_fired(sub_key) + self.add_weapon_fired(sub_key) + elif sub_key == 'scoreboard-score' and int(sub_value) != 0: + player_nonzero_score = True + elif sub_key == 'scoreboard-fastest': + player_fastest = True elif key == 'n': player[key] = unicode(value, 'utf-8') elif key in player_keys: @@ -121,18 +160,25 @@ class Submission(object): self.q.appendleft("{} {}".format(key, value)) break - played = played_in_game(player) - human = is_real_player(player) + played = self.played_in_game(player) + human = self.is_human_player(player) if played and human: self.humans.append(player) if player_fired_weapon: self.human_fired_weapon = True + + if player_nonzero_score: + self.human_nonzero_score = True + + if player_fastest: + self.human_fastest = True + elif played and not human: self.bots.append(player) - else: - self.players.append(player) + + self.players.append(player) def parse_team(self, key, tid): """Construct a team events listing from the submission.""" @@ -152,21 +198,47 @@ class Submission(object): (key, value) = self.next_item() if key is None and value is None: continue + elif key == 'V': + self.version = value + elif key == 'R': + self.revision = value + elif key == 'G': + self.game_type_cd = value + elif key == 'O': + self.mod = value + elif key == 'M': + self.map_name = value + elif key == 'I': + self.match_id = value elif key == 'S': - self.meta[key] = unicode(value, 'utf-8') - elif key == 'P': - self.parse_player(key, value) + self.server_name = unicode(value, 'utf-8') + elif key == 'C': + self.impure_cvar_changes = int(value) + elif key == 'U': + self.port_number = int(value) + elif key == 'D': + self.duration = datetime.timedelta(seconds=int(round(float(value)))) + elif key == 'L': + self.ladder = value elif key == 'Q': self.parse_team(key, value) + elif key == 'P': + self.parse_player(key, value) else: - self.meta[key] = value + raise Exception("Invalid submission") return self + def __repr__(self): + """Debugging representation of a submission.""" + return "game_type_cd: {}, mod: {}, players: {}, humans: {}, bots: {}, weapons: {}".format( + self.game_type_cd, self.mod, len(self.players), len(self.humans), len(self.bots), + self.weapons) + def elo_submission_category(submission): """Determines the Elo category purely by what is in the submission data.""" - mod = submission.meta.get("O", "None") + mod = submission.mod vanilla_allowed_weapons = {"shotgun", "devastator", "blaster", "mortar", "vortex", "electro", "arc", "hagar", "crylink", "machinegun"} @@ -188,79 +260,9 @@ def elo_submission_category(submission): return "general" -def parse_stats_submission(body): +def is_blank_game(submission): """ - Parses the POST request body for a stats submission - """ - # storage vars for the request body - game_meta = {} - events = {} - players = [] - teams = [] - - # we're not in either stanza to start - in_P = in_Q = False - - for line in body.split('\n'): - try: - (key, value) = line.strip().split(' ', 1) - - # Server (S) and Nick (n) fields can have international characters. - if key in 'S' 'n': - value = unicode(value, 'utf-8') - - if key not in 'P' 'Q' 'n' 'e' 't' 'i': - game_meta[key] = value - - if key == 'Q' or key == 'P': - #log.debug('Found a {0}'.format(key)) - #log.debug('in_Q: {0}'.format(in_Q)) - #log.debug('in_P: {0}'.format(in_P)) - #log.debug('events: {0}'.format(events)) - - # check where we were before and append events accordingly - if in_Q and len(events) > 0: - #log.debug('creating a team (Q) entry') - teams.append(events) - events = {} - elif in_P and len(events) > 0: - #log.debug('creating a player (P) entry') - players.append(events) - events = {} - - if key == 'P': - #log.debug('key == P') - in_P = True - in_Q = False - elif key == 'Q': - #log.debug('key == Q') - in_P = False - in_Q = True - - events[key] = value - - if key == 'e': - (subkey, subvalue) = value.split(' ', 1) - events[subkey] = subvalue - if key == 'n': - events[key] = value - if key == 't': - events[key] = value - except: - # no key/value pair - move on to the next line - pass - - # add the last entity we were working on - if in_P and len(events) > 0: - players.append(events) - elif in_Q and len(events) > 0: - teams.append(events) - - return (game_meta, players, teams) - - -def is_blank_game(gametype, players): - """Determine if this is a blank game or not. A blank game is either: + Determine if this is a blank game or not. A blank game is either: 1) a match that ended in the warmup stage, where accuracy events are not present (for non-CTS games) @@ -277,40 +279,24 @@ def is_blank_game(gametype, players): 1) a match in which no player made a positive or negative score """ - r = re.compile(r'acc-.*-cnt-fired') - flg_nonzero_score = False - flg_acc_events = False - flg_fastest_lap = False - - for events in players: - if is_real_player(events) and played_in_game(events): - for (key,value) in events.items(): - if key == 'scoreboard-score' and value != 0: - flg_nonzero_score = True - if r.search(key): - flg_acc_events = True - if key == 'scoreboard-fastest': - flg_fastest_lap = True - - if gametype == 'cts': - return not flg_fastest_lap - elif gametype == 'nb': - return not flg_nonzero_score + if submission.game_type_cd == 'cts': + return not submission.human_fastest + elif submission.game_type_cd == 'nb': + return not submission.human_nonzero_score else: - return not (flg_nonzero_score and flg_acc_events) + return not (submission.human_nonzero_score and submission.human_fired_weapon) -def get_remote_addr(request): - """Get the Xonotic server's IP address""" - if 'X-Forwarded-For' in request.headers: - return request.headers['X-Forwarded-For'] - else: - return request.remote_addr +def has_required_metadata(submission): + """Determines if a submission has all the required metadata fields.""" + return (submission.game_type_cd is not None + and submission.map_name is not None + and submission.match_id is not None + and submission.server_name is not None) -def is_supported_gametype(gametype, version): - """Whether a gametype is supported or not""" - is_supported = False +def is_supported_gametype(submission): + """Determines if a submission is of a valid and supported game type.""" # if the type can be supported, but with version constraints, uncomment # here and add the restriction for a specific version below @@ -333,22 +319,31 @@ def is_supported_gametype(gametype, version): 'tdm', ) - if gametype in supported_game_types: - is_supported = True - else: - is_supported = False + is_supported = submission.game_type_cd in supported_game_types # some game types were buggy before revisions, thus this additional filter - if gametype == 'ca' and version <= 5: + if submission.game_type_cd == 'ca' and submission.version <= 5: is_supported = False return is_supported -def do_precondition_checks(request, game_meta, raw_players): - """Precondition checks for ALL gametypes. - These do not require a database connection.""" - if not has_required_metadata(game_meta): +def has_minimum_real_players(settings, submission): + """ + Determines if the submission has enough human players to store in the database. The minimum + setting comes from the config file under the setting xonstat.minimum_real_players. + """ + try: + minimum_required_players = int(settings.get("xonstat.minimum_required_players")) + except: + minimum_required_players = 2 + + return len(submission.human_players) >= minimum_required_players + + +def do_precondition_checks(settings, submission): + """Precondition checks for ALL gametypes. These do not require a database connection.""" + if not has_required_metadata(submission): msg = "Missing required game metadata" log.debug(msg) raise pyramid.httpexceptions.HTTPUnprocessableEntity( @@ -356,9 +351,7 @@ def do_precondition_checks(request, game_meta, raw_players): content_type="text/plain" ) - try: - version = int(game_meta['V']) - except: + if submission.version is None: msg = "Invalid or incorrect game metadata provided" log.debug(msg) raise pyramid.httpexceptions.HTTPUnprocessableEntity( @@ -366,15 +359,15 @@ def do_precondition_checks(request, game_meta, raw_players): content_type="text/plain" ) - if not is_supported_gametype(game_meta['G'], version): - msg = "Unsupported game type ({})".format(game_meta['G']) + if not is_supported_gametype(submission): + msg = "Unsupported game type ({})".format(submission.game_type_cd) log.debug(msg) raise pyramid.httpexceptions.HTTPOk( body=msg, content_type="text/plain" ) - if not has_minimum_real_players(request.registry.settings, raw_players): + if not has_minimum_real_players(settings, submission): msg = "Not enough real players" log.debug(msg) raise pyramid.httpexceptions.HTTPOk( @@ -382,7 +375,7 @@ def do_precondition_checks(request, game_meta, raw_players): content_type="text/plain" ) - if is_blank_game(game_meta['G'], raw_players): + if is_blank_game(submission): msg = "Blank game" log.debug(msg) raise pyramid.httpexceptions.HTTPOk( @@ -391,74 +384,22 @@ def do_precondition_checks(request, game_meta, raw_players): ) -def num_real_players(player_events): - """ - Returns the number of real players (those who played - and are on the scoreboard). - """ - real_players = 0 - - for events in player_events: - if is_real_player(events) and played_in_game(events): - real_players += 1 - - return real_players - - -def has_minimum_real_players(settings, player_events): - """ - Determines if the collection of player events has enough "real" players - to store in the database. The minimum setting comes from the config file - under the setting xonstat.minimum_real_players. - """ - flg_has_min_real_players = True - - try: - minimum_required_players = int( - settings['xonstat.minimum_required_players']) - except: - minimum_required_players = 2 - - real_players = num_real_players(player_events) - - if real_players < minimum_required_players: - flg_has_min_real_players = False - - return flg_has_min_real_players - - -def has_required_metadata(metadata): - """ - Determines if a give set of metadata has enough data to create a game, - server, and map with. - """ - flg_has_req_metadata = True - - if 'G' not in metadata or\ - 'M' not in metadata or\ - 'I' not in metadata or\ - 'S' not in metadata: - flg_has_req_metadata = False - - return flg_has_req_metadata +def get_remote_addr(request): + """Get the Xonotic server's IP address""" + if 'X-Forwarded-For' in request.headers: + return request.headers['X-Forwarded-For'] + else: + return request.remote_addr def should_do_weapon_stats(game_type_cd): """True of the game type should record weapon stats. False otherwise.""" - if game_type_cd in 'cts': - return False - else: - return True + return game_type_cd not in {'cts'} def gametype_elo_eligible(game_type_cd): """True of the game type should process Elos. False otherwise.""" - elo_game_types = ('duel', 'dm', 'ca', 'ctf', 'tdm', 'ka', 'ft') - - if game_type_cd in elo_game_types: - return True - else: - return False + return game_type_cd in {'duel', 'dm', 'ca', 'ctf', 'tdm', 'ka', 'ft'} def register_new_nick(session, player, new_nick): @@ -801,7 +742,7 @@ def create_default_game_stat(session, game_type_cd): return pgstat -def create_game_stat(session, game_meta, game, server, gmap, player, events): +def create_game_stat(session, game, gmap, player, events): """Game stats handler for all game types""" game_type_cd = game.game_type_cd @@ -947,7 +888,7 @@ def create_team_stat(session, game, events): return teamstat -def create_weapon_stats(session, game_meta, game, player, pgstat, events): +def create_weapon_stats(session, version, game, player, pgstat, events): """Weapon stats handler for all game types""" pwstats = [] @@ -955,7 +896,6 @@ def create_weapon_stats(session, game_meta, game, player, pgstat, events): # To counteract this we divide the data by 2 only for # POSTs coming from version 1. try: - version = int(game_meta['V']) if version == 1: is_doubled = True log.debug('NOTICE: found a version 1 request, halving the weapon stats...') @@ -1044,88 +984,66 @@ def submit_stats(request): "----- END REQUEST BODY -----\n\n") (idfp, status) = verify_request(request) - (game_meta, raw_players, raw_teams) = parse_stats_submission(request.body) - revision = game_meta.get('R', 'unknown') - duration = game_meta.get('D', None) - - # only players present at the end of the match are eligible for stats - raw_players = filter(played_in_game, raw_players) - - do_precondition_checks(request, game_meta, raw_players) + submission = Submission(request.body, request.headers) - # the "duel" gametype is fake - if len(raw_players) == 2 \ - and num_real_players(raw_players) == 2 \ - and game_meta['G'] == 'dm': - game_meta['G'] = 'duel' + do_precondition_checks(request.registry.settings, submission) #---------------------------------------------------------------------- # Actual setup (inserts/updates) below here #---------------------------------------------------------------------- session = DBSession() - game_type_cd = game_meta['G'] - # All game types create Game, Server, Map, and Player records # the same way. server = get_or_create_server( - session = session, - hashkey = idfp, - name = game_meta['S'], - revision = revision, - ip_addr = get_remote_addr(request), - port = game_meta.get('U', None), - impure_cvars = game_meta.get('C', 0)) - - gmap = get_or_create_map( - session = session, - name = game_meta['M']) + session=session, + hashkey=idfp, + name=submission.server_name, + revision=submission.revision, + ip_addr=get_remote_addr(request), + port=submission.port_number, + impure_cvars=submission.impure_cvar_changes + ) + + gmap = get_or_create_map(session, submission.map_name) game = create_game( - session = session, - start_dt = datetime.datetime.utcnow(), - server_id = server.server_id, - game_type_cd = game_type_cd, - map_id = gmap.map_id, - match_id = game_meta['I'], - duration = duration, - mod = game_meta.get('O', None)) + session=session, + start_dt=datetime.datetime.utcnow(), + server_id=server.server_id, + game_type_cd=submission.game_type_cd, + map_id=gmap.map_id, + match_id=submission.match_id, + duration=submission.duration, + mod=submission.mod + ) # keep track of the players we've seen player_ids = [] pgstats = [] hashkeys = {} - for events in raw_players: - player = get_or_create_player( - session = session, - hashkey = events['P'], - nick = events.get('n', None)) - - pgstat = create_game_stat(session, game_meta, game, server, - gmap, player, events) + for events in submission.humans + submission.bots: + player = get_or_create_player(session, events['P'], events.get('n', None)) + pgstat = create_game_stat(session, game, gmap, player, events) pgstats.append(pgstat) if player.player_id > 1: - anticheats = create_anticheats(session, pgstat, game, player, events) + create_anticheats(session, pgstat, game, player, events) if player.player_id > 2: player_ids.append(player.player_id) hashkeys[player.player_id] = events['P'] - if should_do_weapon_stats(game_type_cd) and player.player_id > 1: - pwstats = create_weapon_stats(session, game_meta, game, player, - pgstat, events) + if should_do_weapon_stats(submission.game_type_cd) and player.player_id > 1: + create_weapon_stats(session, submission.version, game, player, pgstat, events) # store them on games for easy access game.players = player_ids - for events in raw_teams: - try: - teamstat = create_team_stat(session, game, events) - except Exception as e: - raise e + for events in submission.teams: + create_team_stat(session, game, events) - if server.elo_ind and gametype_elo_eligible(game_type_cd): + if server.elo_ind and gametype_elo_eligible(submission.game_type_cd): ep = EloProcessor(session, game, pgstats) ep.save(session) @@ -1133,20 +1051,20 @@ def submit_stats(request): log.debug('Success! Stats recorded.') # ranks are fetched after we've done the "real" processing - ranks = get_ranks(session, player_ids, game_type_cd) + ranks = get_ranks(session, player_ids, submission.game_type_cd) # plain text response request.response.content_type = 'text/plain' return { - "now" : calendar.timegm(datetime.datetime.utcnow().timetuple()), - "server" : server, - "game" : game, - "gmap" : gmap, - "player_ids" : player_ids, - "hashkeys" : hashkeys, - "elos" : ep.wip, - "ranks" : ranks, + "now": calendar.timegm(datetime.datetime.utcnow().timetuple()), + "server": server, + "game": game, + "gmap": gmap, + "player_ids": player_ids, + "hashkeys": hashkeys, + "elos": ep.wip, + "ranks": ranks, } except Exception as e: