scripts/gender_profession_bias.py

import re
import json

import pandas as pd
import multiprocessing.pool
from spacy.lang.en import English

gender_lexicons = json.load(open("config/gender_lexicons.json", "r"))
profession_lexicons = json.load(open("config/profession_lexicons.json", "r"))

nlp = English()
nlp.add_pipe("sentencizer")


def get_split_text(text):
    doc = nlp(text)
    sentences = [sent for sent in doc.sents]
    return sentences


def compile_regex_patterns(patterns):
    return [
        re.compile(r"\b({})\b".format("|".join(pattern)), flags=re.IGNORECASE)
        for pattern in patterns
    ]


def get_gender_prof_match_details(df_text):
    male_pronouns = gender_lexicons.get("male_pronouns")
    female_pronouns = gender_lexicons.get("female_pronouns")
    professions = profession_lexicons.get("professions")

    male_pronoun_pat, female_pronoun_pat, professions_pat = compile_regex_patterns(
        [male_pronouns, female_pronouns, professions]
    )

    split_text = get_split_text(df_text)

    results = []

    for text in split_text:
        male_pronoun_match = re.findall(male_pronoun_pat, str(text))
        female_pronoun_match = re.findall(female_pronoun_pat, str(text))

        prof_match = re.findall(professions_pat, str(text))

        both_match = "No"

        if len(male_pronoun_match) != 0 and len(prof_match) != 0:
            both_match = "Yes"

        if len(female_pronoun_match) != 0 and len(prof_match) != 0:
            both_match = "Yes"

        male_pronoun_match = ",".join(male_pronoun_match)
        female_pronoun_match = ",".join(female_pronoun_match)

        prof_match = ",".join(prof_match)

        results.append(
            (
                str(text),
                male_pronoun_match,
                female_pronoun_match,
                prof_match,
                both_match,
            )
        )

    return results


def call_multiprocessing_pool(df_text):
    concurrent = 2000
    pool = multiprocessing.pool.ThreadPool(processes=concurrent)
    result_list = pool.map(get_gender_prof_match_details, df_text, chunksize=1)
    pool.close()

    flat_return_list = [item for sublist in result_list for item in sublist]

    cols = ["Split Text", "Male Pronoun", "Female Pronoun", "Profession", "Both Match"]
    return_df = pd.DataFrame(flat_return_list, columns=cols)

    return return_df


def get_statistics(result):
    conditions = {
        "both_gender_prof_match": result["Both Match"].eq("Yes"),
        "count_male_pronoun": result["Male Pronoun"].ne(""),
        "count_female_pronoun": result["Female Pronoun"].ne(""),
        "count_male_pronoun_profession": result["Male Pronoun"].ne("")
        & result["Profession"].ne(""),
        "count_female_pronoun_profession": result["Female Pronoun"].ne("")
        & result["Profession"].ne(""),
    }

    stats = {key: str(value.sum()) for key, value in conditions.items()}

    stats["total_sentence"] = str(len(result))

    return stats


def eval_gender_profession(data):
    data = data[data.columns[0]].str.lower().str.strip()

    result = call_multiprocessing_pool(data)
    stats = get_statistics(result)

    return stats