Skip to content

Commit

Permalink
Reset moderaiton flags
Browse files Browse the repository at this point in the history
  • Loading branch information
BabyChouSr committed Nov 28, 2024
1 parent 2ef314b commit 5b1fa5e
Showing 1 changed file with 9 additions and 3 deletions.
12 changes: 9 additions & 3 deletions fastchat/serve/moderation/moderator.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ def __init__(self):
self.conv_moderation_responses: List[
Dict[str, Dict[str, Union[str, Dict[str, float]]]]
] = []
self.text_flagged = False
self.csam_flagged = False
self.nsfw_flagged = False

def _image_moderation_filter(self, image: Image) -> Tuple[bool, bool]:
"""Function that detects whether image violates moderation policies.
Expand All @@ -34,6 +37,11 @@ def _text_moderation_filter(self, text: str) -> bool:
"""
raise NotImplementedError

def reset_moderation_flags(self):
self.text_flagged = False
self.csam_flagged = False
self.nsfw_flagged = False

def image_and_text_moderation_filter(
self, image: Image, text: str
) -> Dict[str, Dict[str, Union[str, Dict[str, float]]]]:
Expand Down Expand Up @@ -77,9 +85,6 @@ def __init__(self, use_remote_storage: bool = False):
to the moderation API.
"""
super().__init__()
self.text_flagged = False
self.csam_flagged = False
self.nsfw_flagged = False

def _image_moderation_request(
self, image_bytes: bytes, endpoint: str, api_key: str
Expand Down Expand Up @@ -253,6 +258,7 @@ def image_and_text_moderation_filter(
}
"""
print("moderating text: ", text)
self.reset_moderation_flags()
text_flagged_map = self.text_moderation_filter(text, model_list, do_moderation)

if image is not None:
Expand Down

0 comments on commit 5b1fa5e

Please sign in to comment.