From f4b84374c9bc687c2c4b2fb3b26e2fa65e4085ff Mon Sep 17 00:00:00 2001 From: Mike Hennessy Date: Fri, 30 Apr 2021 09:01:12 -0400 Subject: [PATCH] Use dataclasses for Destiny API responses --- seraphsix/cogs/clan.py | 101 +++-- seraphsix/cogs/server.py | 7 +- seraphsix/cogs/utils/helpers.py | 9 +- seraphsix/constants.py | 1 + seraphsix/models/destiny.py | 718 ++++++++++++++++++++++++++++---- seraphsix/tasks/activity.py | 62 +-- seraphsix/tasks/clan.py | 18 +- seraphsix/tasks/core.py | 26 +- 8 files changed, 773 insertions(+), 169 deletions(-) diff --git a/seraphsix/cogs/clan.py b/seraphsix/cogs/clan.py index 11a573f..771c63a 100644 --- a/seraphsix/cogs/clan.py +++ b/seraphsix/cogs/clan.py @@ -12,11 +12,14 @@ from seraphsix import constants from seraphsix.cogs.utils.checks import is_clan_admin, is_valid_game_mode, is_registered, clan_is_linked -from seraphsix.cogs.utils.helpers import destiny_date_as_utc, date_as_string, get_requestor +from seraphsix.cogs.utils.helpers import date_as_string, get_requestor from seraphsix.cogs.utils.message_manager import MessageManager from seraphsix.cogs.utils.paginator import FieldPages, EmbedPages from seraphsix.database import Member, ClanMember, Clan, Guild, ClanMemberApplication from seraphsix.errors import InvalidAdminError, InvalidCommandError +from seraphsix.models.destiny import ( + DestinyMembershipResponse, DestinyMemberGroupResponse, DestinyGroupResponse, DestinyGroupPendingMembersResponse +) from seraphsix.tasks.activity import get_game_counts, get_last_active from seraphsix.tasks.core import execute_pydest, get_primary_membership, execute_pydest_auth from seraphsix.tasks.clan import info_sync, member_sync @@ -86,17 +89,18 @@ async def get_bungie_details(self, username, bungie_id=None, platform_id=None): if bungie_id: try: player = await execute_pydest( - self.bot.destiny.api.get_membership_data_by_id, bungie_id + self.bot.destiny.api.get_membership_data_by_id, bungie_id, + return_type=DestinyMembershipResponse ) except pydest.PydestException as e: log_message = f"Could not find Destiny player for {username}" log.error(f"{log_message}\n\n{e}\n\n{player}") raise InvalidCommandError(log_message) - for membership in player.response['destinyMemberships']: - if membership['membershipType'] != constants.PLATFORM_BUNGIE: - membership_id = membership['membershipId'] - platform_id = membership['membershipType'] + for membership in player.response.destiny_memberships: + if membership.membership_type != constants.PLATFORM_BUNGIE: + membership_id = membership.membership_id + platform_id = membership.membership_type break else: player = await execute_pydest( @@ -109,28 +113,29 @@ async def get_bungie_details(self, username, bungie_id=None, platform_id=None): if len(player.response) == 1: membership = player.response[0] - if membership['displayName'].lower() == username_lower and membership['membershipType'] == platform_id: - membership_id = membership['membershipId'] - platform_id = membership['membershipType'] + if membership.display_name.lower() == username_lower and membership.membership_type == platform_id: + membership_id = membership.membership_id + platform_id = membership.membership_type else: membership_orig = membership profile = await execute_pydest( - self.bot.destiny.api.get_membership_data_by_id, membership['membershipId'] + self.bot.destiny.api.get_membership_data_by_id, membership.membership_id, + return_type=DestinyMembershipResponse ) - for membership in profile.response['destinyMemberships']: - if membership['displayName'].lower() == username_lower: + for membership in profile.response.destiny_memberships: + if membership.display_name.lower() == username_lower: user_matches = True break if user_matches: - membership_id = membership_orig['membershipId'] - platform_id = membership_orig['membershipType'] + membership_id = membership_orig.membership_id + platform_id = membership_orig.membership_type else: for membership in player.response: - display_name = membership['displayName'].lower() - membership_type = membership['membershipType'] + display_name = membership.display_name.lower() + membership_type = membership.membership_type if membership_type == platform_id and display_name == username_lower: - membership_id = membership['membershipId'] - platform_id = membership['membershipType'] + membership_id = membership.membership_id + platform_id = membership.membership_type break return membership_id, platform_id @@ -145,13 +150,14 @@ async def create_application_embed(self, ctx, requestor_db, guild_db): group_id = None group_name = None groups_info = await execute_pydest( - self.bot.destiny.api.get_groups_for_member, platform_id, membership_id + self.bot.destiny.api.get_groups_for_member, platform_id, membership_id, + return_type=DestinyMemberGroupResponse ) - if len(groups_info.response['results']) > 0: - for group in groups_info.response['results']: - if group['member']['destinyUserInfo']['membershipId'] == membership_id: - group_id = group['group']['groupId'] - group_name = group['group']['name'] + if len(groups_info.response.results) > 0: + for group in groups_info.response.results: + if group.member.destiny_user_info.membership_id == membership_id: + group_id = group.group.group_id + group_name = group.group.name if group_id and group_name: group_url = f'https://www.bungie.net/en/ClanV2/Index?groupId={group_id}' @@ -284,7 +290,8 @@ async def info(self, ctx, *args): embeds = pickle.loads(clan_info_redis) else: for clan_db in clan_dbs: - group = await execute_pydest(self.bot.destiny.api.get_group, clan_db.clan_id) + group = await execute_pydest( + self.bot.destiny.api.get_group, clan_db.clan_id, return_type=DestinyGroupResponse) if not group.response: log.error( f"Could not get details for clan {clan_db.name} ({clan_db.clan_id}) - " @@ -296,26 +303,26 @@ async def info(self, ctx, *args): embed = discord.Embed( colour=constants.BLUE, - title=group['detail']['motto'], - description=group['detail']['about'] + title=group.detail.motto, + description=group.detail.about ) embed.set_author( - name=f"{group['detail']['name']} [{group['detail']['clanInfo']['clanCallsign']}]", + name=f"{group.detail.name} [{group.detail.clan_info.clan_callsign}]", url=f"https://www.bungie.net/en/ClanV2?groupid={clan_db.clan_id}" ) embed.add_field( name="Members", - value=group['detail']['memberCount'], + value=group.detail.member_count, inline=True ) embed.add_field( name="Founder", - value=group['founder']['bungieNetUserInfo']['displayName'], + value=group.founder.bungie_net_user_info.display_name, inline=True ) embed.add_field( name="Founded", - value=date_as_string(destiny_date_as_utc(group['detail']['creationDate']), with_tz=True), + value=date_as_string(group.detail.creation_date), inline=True ) embeds.append(embed) @@ -392,7 +399,8 @@ async def pending(self, ctx): admin_db, manager, group_id=clan_db.clan_id, - access_token=admin_db.bungie_access_token + access_token=admin_db.bungie_access_token, + return_type=DestinyGroupPendingMembersResponse ) embed = discord.Embed( @@ -400,14 +408,14 @@ async def pending(self, ctx): title=f"Pending Clan Members in {clan_db.name}" ) - if len(members.response['results']) == 0: + if len(members.response.results) == 0: embed.description = "None" else: - for member in members.response['results']: - bungie_name = member['destinyUserInfo']['displayName'] - bungie_member_id = member['destinyUserInfo']['membershipId'] - bungie_member_type = member['destinyUserInfo']['membershipType'] - date_applied = date_as_string(destiny_date_as_utc(member['creationDate']), with_tz=True) + for member in members.response.results: + bungie_name = member.destiny_user_info.display_name + bungie_member_id = member.destiny_user_info.membership_id + bungie_member_type = member.destiny_user_info.membership_type + date_applied = date_as_string(member.creation_date, with_tz=True) bungie_url = f"https://www.bungie.net/en/Profile/{bungie_member_type}/{bungie_member_id}" member_info = f"Date Applied: {date_applied}\nProfile: {bungie_url}" embed.add_field(name=bungie_name, value=member_info) @@ -490,7 +498,8 @@ async def invited(self, ctx): admin_db, manager, group_id=clan_db.clan_id, - access_token=admin_db.bungie_access_token + access_token=admin_db.bungie_access_token, + return_type=DestinyGroupPendingMembersResponse ) embed = discord.Embed( @@ -498,16 +507,16 @@ async def invited(self, ctx): title=f"Invited Clan Members in {clan_db.name}" ) - if len(members.response['results']) == 0: + if len(members.response.results) == 0: embed.description = "None" else: - for member in members.response['results']: - bungie_name = member['destinyUserInfo']['displayName'] - bungie_member_id = member['destinyUserInfo']['membershipId'] - bungie_member_type = member['destinyUserInfo']['membershipType'] - date_applied = date_as_string(destiny_date_as_utc(member['creationDate']), with_tz=True) + for member in members.response.results: + bungie_name = member.destiny_user_info.display_name + bungie_member_id = member.destiny_user_info.membership_id + bungie_member_type = member.destiny_user_info.membership_type + date_invited = date_as_string(member.creation_date, with_tz=True) bungie_url = f"https://www.bungie.net/en/Profile/{bungie_member_type}/{bungie_member_id}" - member_info = f"Date Invited: {date_applied}\nProfile: {bungie_url}" + member_info = f"Date Invited: {date_invited}\nProfile: {bungie_url}" embed.add_field(name=bungie_name, value=member_info) await manager.send_embed(embed) diff --git a/seraphsix/cogs/server.py b/seraphsix/cogs/server.py index 060eb3c..370d3ea 100644 --- a/seraphsix/cogs/server.py +++ b/seraphsix/cogs/server.py @@ -7,6 +7,7 @@ from seraphsix.cogs.utils.checks import twitter_enabled, clan_is_linked from seraphsix.cogs.utils.message_manager import MessageManager from seraphsix.database import TwitterChannel, Clan, Guild, Role +from seraphsix.models.destiny import DestinyGroupResponse from seraphsix.tasks.core import execute_pydest from seraphsix.tasks.discord import store_sherpas @@ -83,9 +84,9 @@ async def clanlink(self, ctx, clan_id=None): if not clan_id: return await manager.send_and_clean("Command must include the Destiny clan ID") - group = await execute_pydest(self.bot.destiny.api.get_group, clan_id) - clan_name = group.response['detail']['name'] - callsign = group.response['detail']['clanInfo']['clanCallsign'] + group = await execute_pydest(self.bot.destiny.api.get_group, clan_id, return_type=DestinyGroupResponse) + clan_name = group.response.detail.name + callsign = group.response.detail.clan_info.clan_callsign try: clan_db = await self.bot.database.get(Clan, clan_id=clan_id) diff --git a/seraphsix/cogs/utils/helpers.py b/seraphsix/cogs/utils/helpers.py index 8ce3bcd..298f25e 100644 --- a/seraphsix/cogs/utils/helpers.py +++ b/seraphsix/cogs/utils/helpers.py @@ -4,7 +4,7 @@ from collections import OrderedDict from datetime import datetime from peewee import DoesNotExist -from seraphsix.constants import DESTINY_DATE_FORMAT, DESTINY_DATE_FORMAT_MS, DATE_FORMAT, DATE_FORMAT_TZ +from seraphsix.constants import DATE_FORMAT, DATE_FORMAT_TZ from seraphsix.database import Member, ClanMember, Clan, Guild @@ -47,13 +47,6 @@ def string_to_date(date, date_format=DATE_FORMAT): return datetime.strptime(date, date_format).astimezone(tz=pytz.utc) -def destiny_date_as_utc(date): - try: - return string_to_date(date, DESTINY_DATE_FORMAT_MS) - except ValueError: - return string_to_date(date, DESTINY_DATE_FORMAT) - - def get_timezone_name(timezone, country_code): set_zones = set() # See if it's already a valid 'long' time zone name diff --git a/seraphsix/constants.py b/seraphsix/constants.py index d789bcb..8141bea 100644 --- a/seraphsix/constants.py +++ b/seraphsix/constants.py @@ -259,6 +259,7 @@ DESTINY_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S%z' DESTINY_DATE_FORMAT_MS = '%Y-%m-%dT%H:%M:%S.%f%z' +DESTINY_DATE_FORMAT_API = '%Y-%m-%dT%H:%M:%SZ' FORSAKEN_RELEASE = datetime.strptime('2018-09-04T18:00:00Z', DESTINY_DATE_FORMAT).astimezone(tz=pytz.utc) SHADOWKEEP_RELEASE = datetime.strptime('2019-10-01T18:00:00Z', DESTINY_DATE_FORMAT).astimezone(tz=pytz.utc) BEYOND_LIGHT_RELEASE = datetime.strptime('2020-11-10T18:00:00Z', DESTINY_DATE_FORMAT).astimezone(tz=pytz.utc) diff --git a/seraphsix/models/destiny.py b/seraphsix/models/destiny.py index 4b00797..9bd7b09 100644 --- a/seraphsix/models/destiny.py +++ b/seraphsix/models/destiny.py @@ -1,12 +1,603 @@ -from datetime import datetime +from datetime import datetime, timezone from dataclasses import dataclass, field -from dataclasses_json import dataclass_json, config -from typing import Optional +from dataclasses_json import dataclass_json, config, LetterCase +from marshmallow import fields +from typing import Optional, List, Dict, Any + from seraphsix import constants -from seraphsix.cogs.utils.helpers import destiny_date_as_utc from seraphsix.tasks.parsing import member_hash, member_hash_db +def encode_datetime(obj): + return obj.strftime(constants.DESTINY_DATE_FORMAT_API) + + +def decode_datetime(obj): + try: + return datetime.strptime(obj, constants.DESTINY_DATE_FORMAT) + except ValueError: + return datetime.strptime(obj, constants.DESTINY_DATE_FORMAT_MS) + + +def encode_datetime_timestamp(obj): + return str(int(datetime.timestamp(obj))) + + +def decode_datetime_timestamp(obj): + return datetime.fromtimestamp(float(obj)).astimezone(tz=timezone.utc) + + +def encode_id_string(obj): + return str(obj) + + +def decode_id_string(obj): + if obj: + return int(obj) + else: + return None + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyUserInfo: + cross_save_override: int + applicable_membership_types: List[int] + is_public: bool + membership_type: int + membership_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + last_seen_display_name: Optional[str] = field( + metadata=config(field_name='LastSeenDisplayName'), + default=None + ) + last_seen_display_name_type: Optional[int] = field( + metadata=config(field_name='LastSeenDisplayNameType'), + default=None + ) + display_name: Optional[str] = None + icon_path: Optional[str] = None + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyBungieNetUserInfo: + supplemental_display_name: str + icon_path: str + cross_save_override: int + is_public: bool + membership_type: int + membership_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + display_name: str + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyProfileData: + user_info: DestinyUserInfo + date_last_played: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + versions_owned: int + character_ids: List[int] + season_hashes: List[int] + current_season_hash: int + current_season_reward_power_cap: int + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyProfile: + data: DestinyProfileData + privacy: int + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyProfileResponse: + profile: DestinyProfile + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyCharacterData: + membership_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + membership_type: int + character_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + date_last_played: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + minutes_played_this_session: str + minutes_played_total: str + light: int + stats: Dict[str, int] + race_hash: int + gender_hash: int + class_hash: int + race_type: int + class_type: int + gender_type: int + emblem_path: str + emblem_background_path: str + emblem_hash: int + emblem_color: Dict[str, int] + level_progression: Dict[str, int] + base_character_level: int + percent_to_next_level: int + title_record_hash: int + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyCharacter: + data: Dict[int, DestinyCharacterData] + privacy: int + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyCharacterResponse: + characters: DestinyCharacter + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyActivityStatValue: + value: float + display_value: str + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyActivityStat: + basic: DestinyActivityStatValue + stat_id: Optional[str] = None + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyActivityDetails: + reference_id: int + director_activity_hash: int + instance_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + mode: int + modes: List[int] + is_private: bool + membership_type: int + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyActivity: + period: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + activity_details: DestinyActivityDetails + values: Dict[str, DestinyActivityStat] + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyActivityResponse: + activities: List[DestinyActivity] + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyPlayer: + destiny_user_info: DestinyUserInfo + character_class: str + class_hash: int + race_hash: int + gender_hash: int + character_level: int + light_level: int + emblem_hash: int + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyPGCRWeapon: + reference_id: int + values: Dict[str, DestinyActivityStat] + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyPGCRExtended: + values: Dict[str, DestinyActivityStat] + weapons: Optional[List[DestinyPGCRWeapon]] = None + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyPGCREntry: + standing: int + player: DestinyPlayer + score: DestinyActivityStat + character_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + values: Dict[str, DestinyActivityStat] + extended: DestinyPGCRExtended + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyPGCR: + period: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + activity_details: DestinyActivityDetails + starting_phase_index: int + entries: List[DestinyPGCREntry] + teams: List[Any] + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyMembership: + # Basically the same as DestinyUserInfo with the addition of the "LastSeen" fields + # TODO Consolidate these two? + last_seen_display_name: str = field(metadata=config(field_name='LastSeenDisplayName')) + last_seen_display_name_type: int = field(metadata=config(field_name='LastSeenDisplayNameType')) + icon_path: str + cross_save_override: int + applicable_membership_types: List[int] + is_public: bool + membership_type: int + membership_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + display_name: str + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyBungieNetUser: + membership_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + unique_name: str + display_name: str + profile_picture: int + profile_theme: int + user_title: int + success_message_flags: str + is_deleted: bool + about: str + first_access: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + last_update: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + show_activity: bool + locale: str + locale_inherit_default: bool + show_group_messaging: bool + profile_picture_path: str + profile_theme_name: str + user_title_display: str + status_text: str + status_date: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + psn_display_name: Optional[str] = None + xbox_display_name: Optional[str] = None + steam_display_name: Optional[str] = None + stadia_display_name: Optional[str] = None + twitch_display_name: Optional[str] = None + blizzard_display_name: Optional[str] = None + fb_display_name: Optional[str] = None + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyMembershipResponse: + destiny_memberships: List[DestinyMembership] + bungie_net_user: Optional[DestinyBungieNetUser] = None + primary_membership_id: Optional[int] = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ), + default=None + ) + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupMember: + member_type: int + is_online: bool + last_online_status_change: datetime = field( + metadata=config( + encoder=encode_datetime_timestamp, + decoder=decode_datetime_timestamp, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + group_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + destiny_user_info: DestinyUserInfo + join_date: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + bungie_net_user_info: Optional[DestinyBungieNetUserInfo] = None + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupMembersResponse: + results: List[DestinyGroupMember] + total_results: int + has_more: bool + query: Dict[str, int] + use_total_results: bool + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupFeatures: + maximum_members: int + maximum_memberships_of_group_type: int + capabilities: int + membership_types: List[int] + invite_permission_override: bool + update_culture_permission_override: bool + host_guidedGame_permission_override: int + update_banner_permission_override: bool + join_level: int + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupD2ClanProgression: + progression_hash: int + daily_progress: int + daily_limit: int + weekly_progress: int + weekly_limit: int + current_progress: int + level: int + level_cap: int + step_index: int + progress_to_next_level: int + next_level_at: int + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupClanBannerData: + decal_id: int + decal_color_id: int + decal_background_color_id: int + gonfalon_id: int + gonfalon_color_id: int + gonfalon_detail_id: int + gonfalon_detail_color_id: int + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupClanInfo: + d2_clan_progressions: Dict[int, DestinyGroupD2ClanProgression] + clan_callsign: str + clan_banner_data: DestinyGroupClanBannerData + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupDetail: + group_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + name: str + group_type: int + membership_id_created: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + creation_date: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + modification_date: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + about: str + tags: List[Any] + member_count: int + is_public: bool + is_public_topic_admin_only: bool + motto: str + allow_chat: bool + is_default_post_public: bool + chat_security: int + locale: str + avatar_image_index: int + homepage: int + membership_option: int + default_publicity: int + theme: str + banner_path: str + avatar_path: str + conversation_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + enable_invitation_messaging_for_admins: bool + ban_expire_date: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + features: DestinyGroupFeatures + clan_info: DestinyGroupClanInfo + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupResponse: + detail: DestinyGroupDetail + founder: DestinyGroupMember + allied_ids: List[int] + alliance_status: int + group_join_invite_count: int + current_user_memberships_inactive_for_destiny: bool + current_user_member_map: Dict[Any, Any] + current_user_potential_member_map: Dict[Any, Any] + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyMemberGroup: + member: DestinyGroupMember + group: DestinyGroupDetail + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyMemberGroupResponse: + results: List[DestinyMemberGroup] + total_results: int + has_more: bool + query: Dict[str, int] + use_total_results: bool + are_all_memberships_inactive: Dict[str, bool] + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupPendingMember: + group_id: int = field( + metadata=config( + encoder=encode_id_string, + decoder=decode_id_string, + mm_field=fields.Integer() + ) + ) + creation_date: datetime = field( + metadata=config( + encoder=encode_datetime, + decoder=decode_datetime, + mm_field=fields.DateTime(format=constants.DESTINY_DATE_FORMAT) + ) + ) + resolve_state: int + destiny_user_info: DestinyUserInfo + bungie_net_user_info: Optional[DestinyBungieNetUserInfo] = None + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class DestinyGroupPendingMembersResponse: + results: List[DestinyGroupPendingMember] + total_results: int + has_more: bool + query: Dict[str, int] + use_total_results: bool + + @dataclass_json @dataclass class DestinyResponse: @@ -23,7 +614,7 @@ class DestinyResponse: class DestinyTokenResponse: access_token: str expires_in: int - membership_id: str + membership_id: int refresh_token: str refresh_expires_in: int token_type: str @@ -44,8 +635,8 @@ def __init__(self): self.username = None def __call__(self, details): - self.id = int(details['membershipId']) - self.username = details['displayName'] + self.id = details.membership_id + self.username = details.display_name def __repr__(self): return f"<{type(self).__name__}: {self.username}-{self.id}>" @@ -64,37 +655,38 @@ def __init__(self): def __init__(self, details): self.memberships = self.Memberships() - self.primary_membership_id = details.get('primaryMembershipId') + self.primary_membership_id = details.primary_membership_id self.is_cross_save = self.primary_membership_id is not None - # self.avatar = details.get('profilePicturePath') # TODO - if details.get('destinyUserInfo'): - self._process_membership(details['destinyUserInfo']) - elif details.get('destinyMemberships'): - for entry in details['destinyMemberships']: + if hasattr(details, 'destiny_user_info'): + self._process_membership(details.destiny_user_info) + elif hasattr(details, 'destiny_memberships'): + for entry in details.destiny_memberships: self._process_membership(entry) - if details.get('bungieNetUserInfo'): - self._process_membership(details['bungieNetUserInfo']) + if hasattr(details, 'bungie_net_user_info'): + self._process_membership(details.bungie_net_user_info) - if details.get('bungieNetUser'): - self._process_membership(details['bungieNetUser']) + if hasattr(details, 'bungie_net_user'): + self._process_membership(details.bungie_net_user) def _process_membership(self, entry): - if 'membershipType' not in entry.keys(): + if not hasattr(entry, 'membership_id'): + return + if not hasattr(entry, 'membership_type'): self.memberships.bungie(entry) else: - if entry['membershipType'] == constants.PLATFORM_XBOX: + if entry.membership_type == constants.PLATFORM_XBOX: self.memberships.xbox(entry) - elif entry['membershipType'] == constants.PLATFORM_PSN: + elif entry.membership_type == constants.PLATFORM_PSN: self.memberships.psn(entry) - elif entry['membershipType'] == constants.PLATFORM_BLIZZARD: + elif entry.membership_type == constants.PLATFORM_BLIZZARD: self.memberships.blizzard(entry) - elif entry['membershipType'] == constants.PLATFORM_STEAM: + elif entry.membership_type == constants.PLATFORM_STEAM: self.memberships.steam(entry) - elif entry['membershipType'] == constants.PLATFORM_STADIA: + elif entry.membership_type == constants.PLATFORM_STADIA: self.memberships.stadia(entry) - elif entry['membershipType'] == constants.PLATFORM_BUNGIE: + elif entry.membership_type == constants.PLATFORM_BUNGIE: self.memberships.bungie(entry) def to_dict(self): @@ -120,11 +712,11 @@ class Member(User): def __init__(self, details, user_details): super().__init__(user_details) - self.join_date = destiny_date_as_utc(details['joinDate']) - self.is_online = details['isOnline'] - self.last_online_status_change = datetime.utcfromtimestamp(int(details['lastOnlineStatusChange'])) - self.group_id = int(details['groupId']) - self.member_type = details['memberType'] + self.join_date = details.join_date + self.is_online = details.is_online + self.last_online_status_change = details.last_online_status_change + self.group_id = details.group_id + self.member_type = details.member_type if self.memberships.xbox.id: self.platform_id = constants.PLATFORM_XBOX @@ -150,30 +742,19 @@ def __str__(self): class Player(object): - def __init__(self, details, api=True): - if not api: - self.membership_id = details['membership_id'] - self.membership_type = details['membership_type'] - self.completed = details['completed'] - self.name = details['name'] - self.time_played = details['time_played'] - else: - self.membership_id = details['player']['destinyUserInfo']['membershipId'] - self.membership_type = details['player']['destinyUserInfo']['membershipType'] - - self.completed = False - if details['values']['completed']['basic']['displayValue'] == 'Yes': - self.completed = True + def __init__(self, details): + self.membership_id = details.player.destiny_user_info.membership_id + self.membership_type = details.player.destiny_user_info.membership_type + self.name = details.player.destiny_user_info.display_name - try: - self.name = details['player']['destinyUserInfo']['displayName'] - except KeyError: - self.name = None + self.completed = False + if details.values['completed'].basic.display_value == 'Yes': + self.completed = True - try: - self.time_played = details['values']['timePlayedSeconds']['basic']['value'] - except KeyError: - self.time_played = 0.0 + try: + self.time_played = details.values['timePlayedSeconds'].basic.value + except KeyError: + self.time_played = 0.0 def __str__(self): return f"<{type(self).__name__}: {self.membership_type}-{self.membership_id}>" @@ -183,29 +764,22 @@ def __repr__(self): class Game(object): - def __init__(self, details, api=True): - if not api: - self.mode_id = details['mode_id'] - self.instance_id = details['instance_id'] - self.reference_id = details['reference_id'] - self.date = datetime.strptime(details['date'], constants.DESTINY_DATE_FORMAT) - self.players = [Player(player, api=False) for player in details['players']] - else: - self.mode_id = details['activityDetails']['mode'] - if self.mode_id == 0: - modes = details['activityDetails']['modes'] - modes.sort() - try: - self.mode_id = modes[-1] - except IndexError: - pass - self.instance_id = int(details['activityDetails']['instanceId']) - self.reference_id = details['activityDetails']['referenceId'] - self.date = destiny_date_as_utc(details['period']) - self.players = [] + def __init__(self, details): + self.mode_id = details.activity_details.mode + if self.mode_id == 0: + modes = details.activity_details.modes + modes.sort() + try: + self.mode_id = modes[-1] + except IndexError: + pass + self.instance_id = int(details.activity_details.instance_id) + self.reference_id = details.activity_details.reference_id + self.date = details.period + self.players = [] def set_players(self, details): - for entry in details['entries']: + for entry in details.entries: player = Player(entry) self.players.append(player) diff --git a/seraphsix/tasks/activity.py b/seraphsix/tasks/activity.py index ddbbf9a..202d09e 100644 --- a/seraphsix/tasks/activity.py +++ b/seraphsix/tasks/activity.py @@ -5,11 +5,12 @@ from peewee import DoesNotExist, fn from playhouse.shortcuts import dict_to_model from seraphsix import constants -from seraphsix.cogs.utils.helpers import destiny_date_as_utc from seraphsix.database import ClanMember, Game, GameMember, Guild, Member -from seraphsix.models.destiny import Game as GameApi, ClanGame -from seraphsix.tasks.core import execute_pydest, get_cached_members -from seraphsix.tasks.parsing import member_hash, member_hash_db, parse_platform +from seraphsix.models.destiny import ( + Game as GameApi, ClanGame, DestinyProfileResponse, DestinyActivityResponse, DestinyPGCR +) +from seraphsix.tasks.core import execute_pydest, get_cached_members, get_primary_membership +from seraphsix.tasks.parsing import member_hash, member_hash_db log = logging.getLogger(__name__) @@ -21,20 +22,25 @@ async def get_activity_history(ctx, platform_id, member_id, char_id, count=250, activities = [] data = await execute_pydest( - destiny.api.get_activity_history, platform_id, member_id, char_id, count=count, page=page, mode=mode) + destiny.api.get_activity_history, + platform_id, member_id, char_id, count=count, page=page, mode=mode, + return_type=DestinyActivityResponse + ) if data.response: if full_sync: - while 'activities' in data.response: + while data.response: page += 1 if activities: - activities.extend(data.response['activities']) + activities.extend(data.response.activities) else: - activities = data.response['activities'] + activities = data.response.activities data = await execute_pydest( destiny.api.get_activity_history, - platform_id, member_id, char_id, count=count, page=page, mode=mode) + platform_id, member_id, char_id, count=count, page=page, mode=mode, + return_type=DestinyActivityResponse + ) else: - activities = data.response['activities'] + activities = data.response.activities if len(activities) == count: log.debug( f'Activity count for {platform_id}-{member_id} ({char_id}) ' @@ -45,7 +51,7 @@ async def get_activity_history(ctx, platform_id, member_id, char_id, count=250, async def get_pgcr(ctx, activity_id): destiny = ctx['destiny'] - data = await execute_pydest(destiny.api.get_post_game_carnage_report, activity_id) + data = await execute_pydest(destiny.api.get_post_game_carnage_report, activity_id, return_type=DestinyPGCR) return data.response @@ -58,7 +64,7 @@ async def decode_activity(ctx, reference_id): async def get_activity_list(ctx, platform_id, member_id, characters, count, full_sync=False, mode=0): tasks = [ get_activity_history(ctx, platform_id, member_id, character, count, full_sync, mode) - for character in list(characters.keys()) + for character in characters ] activities = await asyncio.gather(*tasks) all_activities = list(itertools.chain.from_iterable(activities)) @@ -68,15 +74,16 @@ async def get_activity_list(ctx, platform_id, member_id, characters, count, full async def get_last_active(ctx, member_db=None, platform_id=None, member_id=None): acct_last_active = None if member_db and not platform_id and not member_id: - platform_id = member_db.platform_id - member_id, _ = parse_platform(member_db.member, platform_id) + platform_id, member_id, _ = get_primary_membership(member_db) profile = await execute_pydest( - ctx['destiny'].api.get_profile, platform_id, member_id, [constants.COMPONENT_PROFILES]) + ctx['destiny'].api.get_profile, platform_id, member_id, [constants.COMPONENT_PROFILES], + return_type=DestinyProfileResponse + ) if not profile.response: log.error(f"Could not get character data for {platform_id}-{member_id}: {profile.message}") else: - acct_last_active = destiny_date_as_utc(profile.response['profile']['data']['dateLastPlayed']) + acct_last_active = profile.response.profile.data.date_last_played log.debug(f"Found last active date for {platform_id}-{member_id}: {acct_last_active}") return acct_last_active @@ -224,13 +231,10 @@ async def store_all_games(ctx, guild_id, guild_name, count=30): async def get_member_activity(ctx, member_db, count=250, full_sync=False, mode=0): - platform_id = member_db.clanmember.platform_id - member_id, member_username = parse_platform(member_db, platform_id) - - try: - characters = await get_characters(ctx, member_id, platform_id) - except (KeyError, TypeError): - log.error(f"Could not get character data for {platform_id}-{member_id}") + platform_id, member_id, _ = get_primary_membership(member_db) + characters = await get_characters(ctx, member_id, platform_id) + if not characters: + log.error(f"Could not get character data for {platform_id}-{member_id} - {characters}") else: return await get_activity_list(ctx, platform_id, member_id, characters, count, full_sync, mode) @@ -238,10 +242,12 @@ async def get_member_activity(ctx, member_db, count=250, full_sync=False, mode=0 async def get_characters(ctx, member_id, platform_id): destiny = ctx['destiny'] retval = None - data = await execute_pydest( - destiny.api.get_profile, platform_id, member_id, [constants.COMPONENT_CHARACTERS]) - if data.response: - retval = data.response['characters']['data'] + profile = await execute_pydest( + destiny.api.get_profile, platform_id, member_id, [constants.COMPONENT_PROFILES], + return_type=DestinyProfileResponse + ) + if profile.response: + retval = profile.response.profile.data.character_ids return retval @@ -323,7 +329,7 @@ async def store_member_history(ctx, member_db_id, guild_id, guild_name, full_syn activities = await get_member_activity(ctx, member_db, count, full_sync, mode) for activity in activities: - activity_id = activity['activityDetails']['instanceId'] + activity_id = activity.activity_details.instance_id await redis_jobs.enqueue_job( 'process_activity', activity, guild_id, guild_name, _job_id=f'process_activity-{activity_id}' ) diff --git a/seraphsix/tasks/clan.py b/seraphsix/tasks/clan.py index 7410e5d..c73636d 100644 --- a/seraphsix/tasks/clan.py +++ b/seraphsix/tasks/clan.py @@ -6,7 +6,9 @@ from seraphsix.cogs.utils.message_manager import MessageManager from seraphsix.database import Member as MemberDb, ClanMember, Clan, ClanMemberApplication from seraphsix.errors import InvalidAdminError -from seraphsix.models.destiny import Member +from seraphsix.models.destiny import ( + Member, DestinyGroupMembersResponse, DestinyMembershipResponse, DestinyGroupResponse +) from seraphsix.tasks.core import execute_pydest, execute_pydest_auth, set_cached_members, get_primary_membership log = logging.getLogger(__name__) @@ -35,11 +37,13 @@ async def sort_members(database, member_list): async def get_all_members(destiny, group_id): - group = await execute_pydest(destiny.api.get_members_of_group, group_id) - group_members = group.response['results'] + group = await execute_pydest( + destiny.api.get_members_of_group, group_id, return_type=DestinyGroupMembersResponse) + group_members = group.response.results for member in group_members: profile = await execute_pydest( - destiny.api.get_membership_data_by_id, member['destinyUserInfo']['membershipId'] + destiny.api.get_membership_data_by_id, member.destiny_user_info.membership_id, + return_type=DestinyMembershipResponse ) yield Member(member, profile.response) @@ -169,9 +173,9 @@ async def info_sync(ctx, guild_id): clan_changes = {} for clan_db in clan_dbs: - group = await execute_pydest(ctx['destiny'].api.get_group, clan_db.clan_id) - bungie_name = group.response['detail']['name'] - bungie_callsign = group.response['detail']['clanInfo']['clanCallsign'] + group = await execute_pydest(ctx['destiny'].api.get_group, clan_db.clan_id, return_type=DestinyGroupResponse) + bungie_name = group.response.detail.name + bungie_callsign = group.response.detail.clan_info.clan_callsign original_name = clan_db.name original_callsign = clan_db.callsign diff --git a/seraphsix/tasks/core.py b/seraphsix/tasks/core.py index d635c32..1b43bc1 100644 --- a/seraphsix/tasks/core.py +++ b/seraphsix/tasks/core.py @@ -28,6 +28,11 @@ async def create_redis_jobs_pool(): ) +async def queue_redis_job(ctx, message, *args, **kwargs): + log.info(f"Queueing task to {message}") + await ctx['redis_jobs'].enqueue_job(*args, **kwargs) + + def backoff_handler(details): if details['wait'] > 30 or details['tries'] > 10: log.debug( @@ -41,6 +46,12 @@ def backoff_handler(details): backoff.expo, (PydestException, asyncio.TimeoutError, BucketFullException), logger=None, on_backoff=backoff_handler) async def execute_pydest(function, *args, **kwargs): retval = None + + if 'return_type' in kwargs: + return_type = kwargs.pop('return_type') + else: + return_type = DestinyResponse + log.debug(f"{function} {args} {kwargs}") async with config.destiny_api_limiter.ratelimit('destiny_api', delay=True): @@ -58,6 +69,8 @@ async def execute_pydest(function, *args, **kwargs): except Exception: raise RuntimeError(f"Cannot parse Destiny API response {data}") else: + if not res: + raise RuntimeError("Unexpected empty response from the Destiny API") if res.error_status != 'Success': # https://bungie-net.github.io/#/components/schemas/Exceptions.PlatformErrorCodes if res.error_status == 'SystemDisabled': @@ -68,8 +81,10 @@ async def execute_pydest(function, *args, **kwargs): raise PrivateHistoryError else: log.error(f"Error running {function} {args} {kwargs} - {res}") - if res.error_status in ['DestinyAccountNotFound']: + if res.error_status not in ['DestinyAccountNotFound']: raise PydestException + if return_type != DestinyResponse and res.response: + res.response = return_type.from_dict(res.response) retval = res log.debug(f"{function} {args} {kwargs} - {res}") return retval @@ -167,8 +182,8 @@ async def wait_for_msg(channel): def member_dbs_to_dict(member_dbs): members = [] for member_db in member_dbs: - member_dict = model_to_dict(member_db.clanmember, recurse=False) - member_dict['member'] = model_to_dict(member_db, recurse=False) + member_dict = model_to_dict(member_db, recurse=False) + member_dict['clanmember'] = model_to_dict(member_db.clanmember, recurse=False) members.append(member_dict) return members @@ -199,7 +214,7 @@ async def set_cached_members(ctx, guild_id, guild_name): return members -def get_primary_membership(member_db): +def get_primary_membership(member_db, restrict_platform_id=None): memberships = [ [constants.PLATFORM_XBOX, member_db.xbox_id, member_db.xbox_username], [constants.PLATFORM_PSN, member_db.psn_id, member_db.psn_username], @@ -213,7 +228,8 @@ def get_primary_membership(member_db): platform_id = membership[0] username = membership[2] break - elif not membership_id and membership[1]: + elif (restrict_platform_id and restrict_platform_id == membership[0]) or \ + (not membership_id and membership[1]): platform_id, membership_id, username = membership break