update

pages/1_Demo_1.py

@@ -0,0 +1,63 @@
+import streamlit as st
+import pandas as pd
+from datasets import load_dataset
+from random import sample
+from utils.metric import Regard
+from utils.model import gpt2
+import os
+
+# Set up the Streamlit interface
+st.title('Gender Bias Analysis in Text Generation')
+
+
+def check_password():
+    def password_entered():
+        if password_input == os.getenv('PASSWORD'):
+            st.session_state['password_correct'] = True
+        else:
+            st.error("Incorrect Password, please try again.")
+
+    password_input = st.text_input("Enter Password:", type="password")
+    submit_button = st.button("Submit", on_click=password_entered)
+
+    if st.session_state.get('password_correct', False):
+        load_and_process_data()
+    else:
+        st.error("Please enter a valid password to access the demo.")
+
+
+def load_and_process_data():
+    st.subheader('Loading and Processing Data')
+    st.write('Loading the BOLD dataset...')
+    bold = load_dataset("AlexaAI/bold", split="train")
+
+    st.write('Sampling 10 female and male American actors...')
+    female_bold = sample([p for p in bold if p['category'] == 'American_actresses'], 10)
+    male_bold = sample([p for p in bold if p['category'] == 'American_actors'], 10)
+
+    male_prompts = [p['prompts'][0] for p in male_bold]
+    female_prompts = [p['prompts'][0] for p in female_bold]
+
+    GPT2 = gpt2()
+
+    st.write('Generating text for male prompts...')
+    male_generation = GPT2.generate_text(male_prompts, pad_token_id=50256, max_length=50, do_sample=False)
+    male_continuations = [gen.replace(prompt, '') for gen, prompt in zip(male_generation, male_prompts)]
+
+    st.write('Generating text for female prompts...')
+    female_generation = GPT2.generate_text(female_prompts, pad_token_id=50256, max_length=50, do_sample=False)
+    female_continuations = [gen.replace(prompt, '') for gen, prompt in zip(female_generation, female_prompts)]
+
+    st.write('Generated {} male continuations'.format(len(male_continuations)))
+    st.write('Generated {} female continuations'.format(len(female_continuations)))
+
+    st.subheader('Sample Generated Texts')
+    st.write('**Male Prompt:**', male_prompts[0])
+    st.write('**Male Continuation:**', male_continuations[0])
+    st.write('**Female Prompt:**', female_prompts[0])
+    st.write('**Female Continuation:**', female_continuations[0])
+
+    regard = Regard("compare")
+    st.write('Computing regard results to compare male and female continuations...')
+    regard_results = regard.compute(data=male_continuations, references=female_continuations)
+    st.write(regard

requirements.txt

@@ -1 +1,3 @@
-openai
+openai
+transformers
+torch==2.0.1

utils/metric.py

@@ -0,0 +1,55 @@
+from collections import defaultdict
+from statistics import mean
+from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
+
+
+
+class Regard:
+
+    def __init__(self, config_name):
+        self.config_name = config_name
+        regard_tokenizer = AutoTokenizer.from_pretrained("sasha/regardv3")
+        regard_model = AutoModelForSequenceClassification.from_pretrained("sasha/regardv3")
+        self.regard_classifier = pipeline(
+            "text-classification", model=regard_model, top_k=4, tokenizer=regard_tokenizer, truncation=True)
+
+    def regard(self,group):
+        group_scores = defaultdict(list)
+        group_regard = self.regard_classifier(group)
+        for pred in group_regard:
+            for pred_score in pred:
+                group_scores[pred_score["label"]].append(pred_score["score"])
+        return group_regard, dict(group_scores)
+
+    def compute(
+        self,
+        data,
+        references=None,
+        aggregation=None,
+    ):
+        if self.config_name == "compare":
+            pred_scores, pred_regard = self.regard(data)
+            ref_scores, ref_regard = self.regard(references)
+            pred_mean = {k: mean(v) for k, v in pred_regard.items()}
+            pred_max = {k: max(v) for k, v in pred_regard.items()}
+            ref_mean = {k: mean(v) for k, v in ref_regard.items()}
+            ref_max = {k: max(v) for k, v in ref_regard.items()}
+            if aggregation == "maximum":
+                return {
+                    "max_data_regard": pred_max,
+                    "max_references_regard": ref_max,
+                }
+            elif aggregation == "average":
+                return {"average_data_regard": pred_mean, "average_references_regard": ref_mean}
+            else:
+                return {"regard_difference": {key: pred_mean[key] - ref_mean.get(key, 0) for key in pred_mean}}
+        else:
+            pred_scores, pred_regard = self.regard(data)
+            pred_mean = {k: mean(v) for k, v in pred_regard.items()}
+            pred_max = {k: max(v) for k, v in pred_regard.items()}
+            if aggregation == "maximum":
+                return {"max_regard": pred_max}
+            elif aggregation == "average":
+                return {"average_regard": pred_mean}
+            else:
+                return {"regard": pred_scores}

utils/model.py

@@ -0,0 +1,19 @@
+from transformers import pipeline, AutoTokenizer
+
+
+class gpt2:
+    def __init__(self,device="cpu"):
+        self.text_generation = pipeline("text-generation", model="gpt2",device=device)
+        self.tokenizer = AutoTokenizer.from_pretrained("gpt2")
+
+    def generate_text(self,**kwargs):
+        results = self.text_generation(**kwargs)
+
+        return [item['generated_text'] for item in results[0]]
+
+    def get_tokenizer(self):
+        return self.tokenizer
+
+if __name__ == '__main__':
+    gpt2 = gpt2()
+    print(gpt2.generate_text(["Hello, how are you?","I am fine, thank you."]))