X-Git-Url: https://de.git.xonotic.org/?a=blobdiff_plain;f=xonstat%2Fviews%2Fsubmission.py;h=631d93b0f3bb3207c46599ea56af761ff5e86348;hb=57e5611fe7a40c3ff52f1a6eb227f0553841f1b7;hp=88c4bb2839de9f72cb2d24de2668de7f23101e9b;hpb=73a175c101c76feed59dcba793d94e3ef1e63f6d;p=xonotic%2Fxonstat.git diff --git a/xonstat/views/submission.py b/xonstat/views/submission.py index 88c4bb2..631d93b 100755 --- a/xonstat/views/submission.py +++ b/xonstat/views/submission.py @@ -1,16 +1,90 @@ import datetime import logging +import os +import pyramid.httpexceptions import re import time from pyramid.response import Response +from sqlalchemy import Sequence from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound +from xonstat.d0_blind_id import d0_blind_id_verify from xonstat.models import * -from xonstat.util import strip_colors +from xonstat.util import strip_colors, qfont_decode log = logging.getLogger(__name__) -def has_minimum_real_players(player_events): +def is_blank_game(players): + """Determine if this is a blank game or not. A blank game is either: + + 1) a match that ended in the warmup stage, where accuracy events are not + present + + 2) a match in which no player made a positive or negative score AND was + on the scoreboard + """ + r = re.compile(r'acc-.*-cnt-fired') + flg_nonzero_score = False + flg_acc_events = False + + for events in players: + if is_real_player(events): + for (key,value) in events.items(): + if key == 'scoreboard-score' and value != '0': + flg_nonzero_score = True + if r.search(key): + flg_acc_events = True + + return not (flg_nonzero_score and flg_acc_events) + +def get_remote_addr(request): + """Get the Xonotic server's IP address""" + if 'X-Forwarded-For' in request.headers: + return request.headers['X-Forwarded-For'] + else: + return request.remote_addr + + +def is_supported_gametype(gametype): + """Whether a gametype is supported or not""" + flg_supported = True + + if gametype == 'cts' or gametype == 'lms': + flg_supported = False + + return flg_supported + + +def verify_request(request): + try: + (idfp, status) = d0_blind_id_verify( + sig=request.headers['X-D0-Blind-Id-Detached-Signature'], + querystring='', + postdata=request.body) + + log.debug('\nidfp: {0}\nstatus: {1}'.format(idfp, status)) + except: + idfp = None + status = None + + return (idfp, status) + + +def num_real_players(player_events, count_bots=False): + """ + Returns the number of real players (those who played + and are on the scoreboard). + """ + real_players = 0 + + for events in player_events: + if is_real_player(events, count_bots): + real_players += 1 + + return real_players + + +def has_minimum_real_players(settings, player_events): """ Determines if the collection of player events has enough "real" players to store in the database. The minimum setting comes from the config file @@ -18,13 +92,16 @@ def has_minimum_real_players(player_events): """ flg_has_min_real_players = True - real_players = 0 - for events in player_events: - if is_real_player(events): - real_players += 1 + try: + minimum_required_players = int( + settings['xonstat.minimum_required_players']) + except: + minimum_required_players = 2 + + real_players = num_real_players(player_events) #TODO: put this into a config setting in the ini file? - if real_players < 1: + if real_players < minimum_required_players: flg_has_min_real_players = False return flg_has_min_real_players @@ -37,19 +114,20 @@ def has_required_metadata(metadata): """ flg_has_req_metadata = True - if 'T' not in game_meta or\ - 'G' not in game_meta or\ - 'M' not in game_meta or\ - 'S' not in game_meta: + if 'T' not in metadata or\ + 'G' not in metadata or\ + 'M' not in metadata or\ + 'I' not in metadata or\ + 'S' not in metadata: flg_has_req_metadata = False return flg_has_req_metadata - -def is_real_player(events): + +def is_real_player(events, count_bots=False): """ Determines if a given set of player events correspond with a player who - + 1) is not a bot (P event does not look like a bot) 2) played in the game (matches 1) 3) was present at the end of the game (scoreboardvalid 1) @@ -58,9 +136,10 @@ def is_real_player(events): """ flg_is_real = False - if not events['P'].startswith('bot'): - # removing 'joins' here due to bug, but it should be here - if 'matches' in events and 'scoreboardvalid' in events: + # removing 'joins' here due to bug, but it should be here + if 'matches' in events and 'scoreboardvalid' in events: + if (events['P'].startswith('bot') and count_bots) or \ + not events['P'].startswith('bot'): flg_is_real = True return flg_is_real @@ -78,53 +157,71 @@ def register_new_nick(session, player, new_nick): # see if that nick already exists stripped_nick = strip_colors(player.nick) try: - player_nick = session.query(PlayerNick).filter_by( - player_id=player.player_id, stripped_nick=stripped_nick).one() + player_nick = session.query(PlayerNick).filter_by( + player_id=player.player_id, stripped_nick=stripped_nick).one() except NoResultFound, e: - # player_id/stripped_nick not found, create one + # player_id/stripped_nick not found, create one # but we don't store "Anonymous Player #N" if not re.search('^Anonymous Player #\d+$', player.nick): - player_nick = PlayerNick() + player_nick = PlayerNick() player_nick.player_id = player.player_id - player_nick.stripped_nick = stripped_nick + player_nick.stripped_nick = player.stripped_nick player_nick.nick = player.nick session.add(player_nick) # We change to the new nick regardless player.nick = new_nick + player.stripped_nick = strip_colors(new_nick) session.add(player) -def get_or_create_server(session=None, name=None): +def get_or_create_server(session=None, name=None, hashkey=None, ip_addr=None, + revision=None): """ Find a server by name or create one if not found. Parameters: session - SQLAlchemy database session factory name - server name of the server to be found or created + hashkey - server hashkey """ try: # find one by that name, if it exists server = session.query(Server).filter_by(name=name).one() - log.debug("Found server id {0}: {1}".format( - server.server_id, server.name.encode('utf-8'))) + + # store new hashkey + if server.hashkey != hashkey: + server.hashkey = hashkey + session.add(server) + + # store new IP address + if server.ip_addr != ip_addr: + server.ip_addr = ip_addr + session.add(server) + + # store new revision + if server.revision != revision: + server.revision = revision + session.add(server) + + log.debug("Found existing server {0}".format(server.server_id)) + + except MultipleResultsFound, e: + # multiple found, so also filter by hashkey + server = session.query(Server).filter_by(name=name).\ + filter_by(hashkey=hashkey).one() + log.debug("Found existing server {0}".format(server.server_id)) + except NoResultFound, e: - server = Server(name=name) + # not found, create one + server = Server(name=name, hashkey=hashkey) session.add(server) session.flush() - log.debug("Created server id {0}: {1}".format( - server.server_id, server.name.encode('utf-8'))) - except MultipleResultsFound, e: - # multiple found, so use the first one but warn - log.debug(e) - servers = session.query(Server).filter_by(name=name).order_by( - Server.server_id).all() - server = servers[0] - log.debug("Created server id {0}: {1} but found \ - multiple".format( - server.server_id, server.name.encode('utf-8'))) + log.debug("Created server {0} with hashkey {1}".format( + server.server_id, server.hashkey)) return server + def get_or_create_map(session=None, name=None): """ Find a map by name or create one if not found. Parameters: @@ -156,7 +253,7 @@ def get_or_create_map(session=None, name=None): def create_game(session=None, start_dt=None, game_type_cd=None, - server_id=None, map_id=None, winner=None): + server_id=None, map_id=None, winner=None, match_id=None): """ Creates a game. Parameters: @@ -167,14 +264,24 @@ def create_game(session=None, start_dt=None, game_type_cd=None, map_id - map on which the game was played winner - the team id of the team that won """ - - game = Game(start_dt=start_dt, game_type_cd=game_type_cd, + seq = Sequence('games_game_id_seq') + game_id = session.execute(seq) + game = Game(game_id=game_id, start_dt=start_dt, game_type_cd=game_type_cd, server_id=server_id, map_id=map_id, winner=winner) - session.add(game) - session.flush() - log.debug("Created game id {0} on server {1}, map {2} at \ - {3}".format(game.game_id, - server_id, map_id, start_dt)) + game.match_id = match_id + + try: + session.query(Game).filter(Game.server_id==server_id).\ + filter(Game.match_id==match_id).one() + # if a game under the same server and match_id found, + # this is a duplicate game and can be ignored + raise pyramid.httpexceptions.HTTPOk('OK') + except NoResultFound, e: + # server_id/match_id combination not found. game is ok to insert + session.add(game) + log.debug("Created game id {0} on server {1}, map {2} at \ + {3}".format(game.game_id, + server_id, map_id, start_dt)) return game @@ -189,7 +296,7 @@ def get_or_create_player(session=None, hashkey=None, nick=None): nick - nick of the player (in case of a first time create) """ # if we have a bot - if re.search('^bot#\d+$', hashkey): + if re.search('^bot#\d+$', hashkey) or re.search('^bot#\d+#', hashkey): player = session.query(Player).filter_by(player_id=1).one() # if we have an untracked player elif re.search('^player#\d+$', hashkey): @@ -199,28 +306,30 @@ def get_or_create_player(session=None, hashkey=None, nick=None): # see if the player is already in the database # if not, create one and the hashkey along with it try: - hashkey = session.query(Hashkey).filter_by( + hk = session.query(Hashkey).filter_by( hashkey=hashkey).one() player = session.query(Player).filter_by( - player_id=hashkey.player_id).one() + player_id=hk.player_id).one() log.debug("Found existing player {0} with hashkey {1}".format( - player.player_id, hashkey.hashkey)) + player.player_id, hashkey)) except: player = Player() session.add(player) session.flush() - # if nick is given to us, use it. If not, use "Anonymous Player" + # if nick is given to us, use it. If not, use "Anonymous Player" # with a suffix added for uniqueness. if nick: player.nick = nick[:128] - else: + player.stripped_nick = strip_colors(nick[:128]) + else: player.nick = "Anonymous Player #{0}".format(player.player_id) + player.stripped_nick = player.nick - hashkey = Hashkey(player_id=player.player_id, hashkey=hashkey) - session.add(hashkey) + hk = Hashkey(player_id=player.player_id, hashkey=hashkey) + session.add(hk) log.debug("Created player {0} ({2}) with hashkey {1}".format( - player.player_id, hashkey.hashkey, player.nick.encode('utf-8'))) + player.player_id, hashkey, player.nick.encode('utf-8'))) return player @@ -238,7 +347,10 @@ def create_player_game_stat(session=None, player=None, # in here setup default values (e.g. if game type is CTF then # set kills=0, score=0, captures=0, pickups=0, fckills=0, etc # TODO: use game's create date here instead of now() - pgstat = PlayerGameStat(create_dt=datetime.datetime.now()) + seq = Sequence('player_game_stats_player_game_stat_id_seq') + pgstat_id = session.execute(seq) + pgstat = PlayerGameStat(player_game_stat_id=pgstat_id, + create_dt=datetime.datetime.utcnow()) # set player id from player record pgstat.player_id = player.player_id @@ -249,7 +361,7 @@ def create_player_game_stat(session=None, player=None, # all games have a score pgstat.score = 0 - if game.game_type_cd == 'dm': + if game.game_type_cd == 'dm' or game.game_type_cd == 'tdm' or game.game_type_cd == 'duel': pgstat.kills = 0 pgstat.deaths = 0 pgstat.suicides = 0 @@ -277,14 +389,14 @@ def create_player_game_stat(session=None, player=None, if key == 'scoreboard-kills': pgstat.kills = value if key == 'scoreboard-suicides': pgstat.suicides = value - # check to see if we had a name, and if - # not use the name from the player id + # check to see if we had a name, and if + # not use an anonymous handle if pgstat.nick == None: - pgstat.nick = player.nick + pgstat.nick = "Anonymous Player" + pgstat.stripped_nick = "Anonymous Player" - # if the nick we end up with is different from the one in the - # player record, change the nick to reflect the new value - if pgstat.nick != player.nick and player.player_id > 1: + # otherwise process a nick change + elif pgstat.nick != player.nick and player.player_id > 2: register_new_nick(session, player, pgstat.nick) # if the player is ranked #1 and it is a team game, set the game's winner @@ -295,7 +407,6 @@ def create_player_game_stat(session=None, player=None, session.add(game) session.add(pgstat) - session.flush() return pgstat @@ -319,7 +430,10 @@ def create_player_weapon_stats(session=None, player=None, matched = re.search("acc-(.*?)-cnt-fired", key) if matched: weapon_cd = matched.group(1) + seq = Sequence('player_weapon_stats_player_weapon_stats_id_seq') + pwstat_id = session.execute(seq) pwstat = PlayerWeaponStat() + pwstat.player_weapon_stats_id = pwstat_id pwstat.player_id = player.player_id pwstat.game_id = game.game_id pwstat.player_game_stat_id = pgstat.player_game_stat_id @@ -361,19 +475,17 @@ def parse_body(request): player_events = {} current_team = None players = [] - - log.debug(request.body) for line in request.body.split('\n'): try: (key, value) = line.strip().split(' ', 1) # Server (S) and Nick (n) fields can have international characters. - # We encode these as UTF-8. + # We convert to UTF-8. if key in 'S' 'n': value = unicode(value, 'utf-8') - - if key in 'V' 'T' 'G' 'M' 'S' 'C' 'R' 'W': + + if key in 'V' 'T' 'G' 'M' 'S' 'C' 'R' 'W' 'I': game_meta[key] = value if key == 'P': @@ -382,7 +494,7 @@ def parse_body(request): if len(player_events) != 0: players.append(player_events) player_events = {} - + player_events[key] = value if key == 'e': @@ -395,7 +507,7 @@ def parse_body(request): except: # no key/value pair - move on to the next line pass - + # add the last player we were working on if len(player_events) > 0: players.append(player_events) @@ -416,7 +528,7 @@ def create_player_stats(session=None, player=None, game=None, create_player_weapon_stats(session=session, player=player, game=game, pgstat=pgstat, player_events=player_events) - + def stats_submit(request): """ @@ -425,26 +537,54 @@ def stats_submit(request): try: session = DBSession() + log.debug("\n----- BEGIN REQUEST BODY -----\n" + request.body + + "----- END REQUEST BODY -----\n\n") + + (idfp, status) = verify_request(request) + if not idfp: + log.debug("ERROR: Unverified request") + raise pyramid.httpexceptions.HTTPUnauthorized("Unverified request") + (game_meta, players) = parse_body(request) - + if not has_required_metadata(game_meta): - log.debug("Required game meta fields (T, G, M, or S) missing. "\ - "Can't continue.") - raise Exception("Required game meta fields (T, G, M, or S) missing.") - - if not has_minimum_real_players(players): - raise Exception("The number of real players is below the minimum. "\ - "Stats will be ignored.") - - server = get_or_create_server(session=session, name=game_meta['S']) + log.debug("ERROR: Required game meta missing") + raise pyramid.httpexceptions.HTTPUnprocessableEntity("Missing game meta") + + if not is_supported_gametype(game_meta['G']): + log.debug("ERROR: Unsupported gametype") + raise pyramid.httpexceptions.HTTPOk("OK") + + if not has_minimum_real_players(request.registry.settings, players): + log.debug("ERROR: Not enough real players") + raise pyramid.httpexceptions.HTTPOk("OK") + + if is_blank_game(players): + log.debug("ERROR: Blank game") + raise pyramid.httpexceptions.HTTPOk("OK") + + # FIXME: if we have two players and game type is 'dm', + # change this into a 'duel' gametype. This should be + # removed when the stats actually send 'duel' instead of 'dm' + if num_real_players(players, count_bots=True) == 2 and \ + game_meta['G'] == 'dm': + game_meta['G'] = 'duel' + + server = get_or_create_server(session=session, hashkey=idfp, + name=game_meta['S'], revision=game_meta['R'], + ip_addr=get_remote_addr(request)) + gmap = get_or_create_map(session=session, name=game_meta['M']) + # FIXME: use the gmtime instead of utcnow() when the timezone bug is + # fixed game = create_game(session=session, - start_dt=datetime.datetime( - *time.gmtime(float(game_meta['T']))[:6]), + start_dt=datetime.datetime.utcnow(), + #start_dt=datetime.datetime( + #*time.gmtime(float(game_meta['T']))[:6]), server_id=server.server_id, game_type_cd=game_meta['G'], - map_id=gmap.map_id, winner=winner) - + map_id=gmap.map_id, match_id=game_meta['I']) + # find or create a record for each player # and add stats for each if they were present at the end # of the game @@ -455,16 +595,22 @@ def stats_submit(request): nick = None if 'matches' in player_events and 'scoreboardvalid' \ - in player_events: + in player_events: player = get_or_create_player(session=session, hashkey=player_events['P'], nick=nick) log.debug('Creating stats for %s' % player_events['P']) create_player_stats(session=session, player=player, game=game, player_events=player_events) - + + # update elos + try: + game.process_elos(session) + except Exception as e: + log.debug('Error (non-fatal): elo processing failed.') + session.commit() log.debug('Success! Stats recorded.') return Response('200 OK') except Exception as e: session.rollback() - raise e + return e