add necessary file

anonymous_demo/__init__.py

@@ -0,0 +1,5 @@
+__version__ = "1.0.0"
+
+__name__ = "anonymous_demo"
+
+from anonymous_demo.functional import TADCheckpointManager

anonymous_demo/core/tad/classic/__bert__/README.MD

@@ -0,0 +1,3 @@
+## This is the simple migration from ABSA-PyTorch under MIT license
+
+Project Address: https://github.com/songyouwei/ABSA-PyTorch

anonymous_demo/core/tad/classic/__bert__/__init__.py

@@ -0,0 +1 @@
+from .models import *

anonymous_demo/core/tad/classic/__bert__/dataset_utils/data_utils_for_inference.py

@@ -0,0 +1,121 @@
+import tqdm
+from findfile import find_cwd_dir
+from torch.utils.data import Dataset
+from transformers import AutoTokenizer
+
+
+class Tokenizer4Pretraining:
+    def __init__(self, max_seq_len, opt, **kwargs):
+        if kwargs.pop("offline", False):
+            self.tokenizer = AutoTokenizer.from_pretrained(
+                find_cwd_dir(opt.pretrained_bert.split("/")[-1]),
+                do_lower_case="uncased" in opt.pretrained_bert,
+            )
+        else:
+            self.tokenizer = AutoTokenizer.from_pretrained(
+                opt.pretrained_bert, do_lower_case="uncased" in opt.pretrained_bert
+            )
+        self.max_seq_len = max_seq_len
+
+    def text_to_sequence(self, text, reverse=False, padding="post", truncating="post"):
+        return self.tokenizer.encode(
+            text,
+            truncation=True,
+            padding="max_length",
+            max_length=self.max_seq_len,
+            return_tensors="pt",
+        )
+
+
+class BERTTADDataset(Dataset):
+    def __init__(self, tokenizer, opt):
+        self.bert_baseline_input_colses = {"bert": ["text_bert_indices"]}
+
+        self.tokenizer = tokenizer
+        self.opt = opt
+        self.all_data = []
+
+    def parse_sample(self, text):
+        return [text]
+
+    def prepare_infer_sample(self, text: str, ignore_error):
+        self.process_data(self.parse_sample(text), ignore_error=ignore_error)
+
+    def process_data(self, samples, ignore_error=True):
+        all_data = []
+        if len(samples) > 100:
+            it = tqdm.tqdm(
+                samples, postfix="preparing text classification inference dataloader..."
+            )
+        else:
+            it = samples
+        for text in it:
+            try:
+                # handle for empty lines in inference datasets
+                if text is None or "" == text.strip():
+                    raise RuntimeError("Invalid Input!")
+
+                if "!ref!" in text:
+                    text, _, labels = text.strip().partition("!ref!")
+                    text = text.strip()
+                    if labels.count(",") == 2:
+                        label, is_adv, adv_train_label = labels.strip().split(",")
+                        label, is_adv, adv_train_label = (
+                            label.strip(),
+                            is_adv.strip(),
+                            adv_train_label.strip(),
+                        )
+                    elif labels.count(",") == 1:
+                        label, is_adv = labels.strip().split(",")
+                        label, is_adv = label.strip(), is_adv.strip()
+                        adv_train_label = "-100"
+                    elif labels.count(",") == 0:
+                        label = labels.strip()
+                        adv_train_label = "-100"
+                        is_adv = "-100"
+                    else:
+                        label = "-100"
+                        adv_train_label = "-100"
+                        is_adv = "-100"
+
+                    label = int(label)
+                    adv_train_label = int(adv_train_label)
+                    is_adv = int(is_adv)
+
+                else:
+                    text = text.strip()
+                    label = -100
+                    adv_train_label = -100
+                    is_adv = -100
+
+                text_indices = self.tokenizer.text_to_sequence("{}".format(text))
+
+                data = {
+                    "text_bert_indices": text_indices[0],
+                    "text_raw": text,
+                    "label": label,
+                    "adv_train_label": adv_train_label,
+                    "is_adv": is_adv,
+                    # 'label': self.opt.label_to_index.get(label, -100) if isinstance(label, str) else label,
+                    #
+                    # 'adv_train_label': self.opt.adv_train_label_to_index.get(adv_train_label, -100) if isinstance(adv_train_label, str) else adv_train_label,
+                    #
+                    # 'is_adv': self.opt.is_adv_to_index.get(is_adv, -100) if isinstance(is_adv, str) else is_adv,
+                }
+
+                all_data.append(data)
+
+            except Exception as e:
+                if ignore_error:
+                    print("Ignore error while processing:", text)
+                else:
+                    raise e
+
+        self.all_data = all_data
+        return self.all_data
+
+    def __getitem__(self, index):
+        return self.all_data[index]
+
+    def __len__(self):
+        return len(self.all_data)

anonymous_demo/core/tad/classic/__bert__/models/__init__.py

@@ -0,0 +1 @@
+from .tad_bert import TADBERT

anonymous_demo/core/tad/classic/__bert__/models/tad_bert.py

@@ -0,0 +1,46 @@
+import torch
+import torch.nn as nn
+from transformers.models.bert.modeling_bert import BertPooler
+
+from anonymous_demo.network.sa_encoder import Encoder
+
+
+class TADBERT(nn.Module):
+    inputs = ["text_bert_indices"]
+
+    def __init__(self, bert, opt):
+        super(TADBERT, self).__init__()
+        self.opt = opt
+        self.bert = bert
+        self.pooler = BertPooler(bert.config)
+        self.dense1 = nn.Linear(self.opt.hidden_dim, self.opt.class_dim)
+        self.dense2 = nn.Linear(self.opt.hidden_dim, self.opt.adv_det_dim)
+        self.dense3 = nn.Linear(self.opt.hidden_dim, self.opt.class_dim)
+
+        self.encoder1 = Encoder(self.bert.config, opt=opt)
+        self.encoder2 = Encoder(self.bert.config, opt=opt)
+        self.encoder3 = Encoder(self.bert.config, opt=opt)
+
+    def forward(self, inputs):
+        text_raw_indices = inputs[0]
+        last_hidden_state = self.bert(text_raw_indices)["last_hidden_state"]
+
+        sent_logits = self.dense1(self.pooler(last_hidden_state))
+        advdet_logits = self.dense2(self.pooler(last_hidden_state))
+        adv_tr_logits = self.dense3(self.pooler(last_hidden_state))
+
+        att_score = torch.nn.functional.normalize(
+            last_hidden_state.abs().sum(dim=1, keepdim=False)
+            - last_hidden_state.abs().min(dim=1, keepdim=True)[0],
+            p=1,
+            dim=1,
+        )
+
+        outputs = {
+            "sent_logits": sent_logits,
+            "advdet_logits": advdet_logits,
+            "adv_tr_logits": adv_tr_logits,
+            "last_hidden_state": last_hidden_state,
+            "att_score": att_score,
+        }
+        return outputs

anonymous_demo/core/tad/models/__init__.py

@@ -0,0 +1,9 @@
+import anonymous_demo.core.tad.classic.__bert__.models
+
+
+class BERTTADModelList(list):
+    TADBERT = anonymous_demo.core.tad.classic.__bert__.TADBERT
+
+    def __init__(self):
+        model_list = [self.TADBERT]
+        super().__init__(model_list)

anonymous_demo/core/tad/prediction/tad_classifier.py

@@ -0,0 +1,518 @@
+import json
+import os
+import pickle
+import time
+
+import torch
+import tqdm
+from findfile import find_file, find_cwd_dir
+from termcolor import colored
+
+from torch.utils.data import DataLoader
+from transformers import (
+    AutoTokenizer,
+    AutoModel,
+    AutoConfig,
+    DebertaV2ForMaskedLM,
+    RobertaForMaskedLM,
+    BertForMaskedLM,
+)
+
+from ....functional.dataset.dataset_manager import detect_infer_dataset
+
+from ..models import BERTTADModelList
+from ..classic.__bert__.dataset_utils.data_utils_for_inference import (
+    BERTTADDataset,
+    Tokenizer4Pretraining,
+)
+
+from ....utils.demo_utils import (
+    print_args,
+    TransformerConnectionError,
+    get_device,
+    build_embedding_matrix,
+)
+
+
+def init_attacker(tad_classifier, defense):
+    try:
+        from textattack import Attacker
+        from textattack.attack_recipes import (
+            BAEGarg2019,
+            PWWSRen2019,
+            TextFoolerJin2019,
+            PSOZang2020,
+            IGAWang2019,
+            GeneticAlgorithmAlzantot2018,
+            DeepWordBugGao2018,
+        )
+        from textattack.datasets import Dataset
+        from textattack.models.wrappers import HuggingFaceModelWrapper
+
+        class DemoModelWrapper(HuggingFaceModelWrapper):
+            def __init__(self, model):
+                self.model = model  # pipeline = pipeline
+
+            def __call__(self, text_inputs, **kwargs):
+                outputs = []
+                for text_input in text_inputs:
+                    raw_outputs = self.model.infer(
+                        text_input, print_result=False, **kwargs
+                    )
+                    outputs.append(raw_outputs["probs"])
+                return outputs
+
+        class SentAttacker:
+            def __init__(self, model, recipe_class=BAEGarg2019):
+                model = model
+                model_wrapper = DemoModelWrapper(model)
+
+                recipe = recipe_class.build(model_wrapper)
+
+                _dataset = [("", 0)]
+                _dataset = Dataset(_dataset)
+
+                self.attacker = Attacker(recipe, _dataset)
+
+        attackers = {
+            "bae": BAEGarg2019,
+            "pwws": PWWSRen2019,
+            "textfooler": TextFoolerJin2019,
+            "pso": PSOZang2020,
+            "iga": IGAWang2019,
+            "ga": GeneticAlgorithmAlzantot2018,
+            "wordbugger": DeepWordBugGao2018,
+        }
+        return SentAttacker(tad_classifier, attackers[defense])
+    except Exception as e:
+        print("Original error:", e)
+
+
+def get_mlm_and_tokenizer(text_classifier, config):
+    if isinstance(text_classifier, TADTextClassifier):
+        base_model = text_classifier.model.bert.base_model
+    else:
+        base_model = text_classifier.bert.base_model
+    pretrained_config = AutoConfig.from_pretrained(config.pretrained_bert)
+    if "deberta-v3" in config.pretrained_bert:
+        MLM = DebertaV2ForMaskedLM(pretrained_config)
+        MLM.deberta = base_model
+    elif "roberta" in config.pretrained_bert:
+        MLM = RobertaForMaskedLM(pretrained_config)
+        MLM.roberta = base_model
+    else:
+        MLM = BertForMaskedLM(pretrained_config)
+        MLM.bert = base_model
+    return MLM, AutoTokenizer.from_pretrained(config.pretrained_bert)
+
+
+class TADTextClassifier:
+    def __init__(self, model_arg=None, cal_perplexity=False, **kwargs):
+        """
+        from_train_model: load inference model from trained model
+        """
+        self.cal_perplexity = cal_perplexity
+        # load from a training
+        if not isinstance(model_arg, str):
+            print("Load text classifier from training")
+            self.model = model_arg[0]
+            self.opt = model_arg[1]
+            self.tokenizer = model_arg[2]
+        else:
+            try:
+                if "fine-tuned" in model_arg:
+                    raise ValueError(
+                        "Do not support to directly load a fine-tuned model, please load a .state_dict or .model instead!"
+                    )
+                print("Load text classifier from", model_arg)
+                state_dict_path = find_file(
+                    model_arg, key=".state_dict", exclude_key=["__MACOSX"]
+                )
+                model_path = find_file(
+                    model_arg, key=".model", exclude_key=["__MACOSX"]
+                )
+                tokenizer_path = find_file(
+                    model_arg, key=".tokenizer", exclude_key=["__MACOSX"]
+                )
+                config_path = find_file(
+                    model_arg, key=".config", exclude_key=["__MACOSX"]
+                )
+
+                print("config: {}".format(config_path))
+                print("state_dict: {}".format(state_dict_path))
+                print("model: {}".format(model_path))
+                print("tokenizer: {}".format(tokenizer_path))
+
+                with open(config_path, mode="rb") as f:
+                    self.opt = pickle.load(f)
+                    self.opt.device = get_device(kwargs.pop("auto_device", True))[0]
+
+                if state_dict_path or model_path:
+                    if hasattr(BERTTADModelList, self.opt.model.__name__):
+                        if state_dict_path:
+                            if kwargs.pop("offline", False):
+                                self.bert = AutoModel.from_pretrained(
+                                    find_cwd_dir(
+                                        self.opt.pretrained_bert.split("/")[-1]
+                                    )
+                                )
+                            else:
+                                self.bert = AutoModel.from_pretrained(
+                                    self.opt.pretrained_bert
+                                )
+                            self.model = self.opt.model(self.bert, self.opt)
+                            self.model.load_state_dict(
+                                torch.load(state_dict_path, map_location="cpu")
+                            )
+                        elif model_path:
+                            self.model = torch.load(model_path, map_location="cpu")
+
+                        try:
+                            self.tokenizer = Tokenizer4Pretraining(
+                                max_seq_len=self.opt.max_seq_len, opt=self.opt, **kwargs
+                            )
+                        except ValueError:
+                            if tokenizer_path:
+                                with open(tokenizer_path, mode="rb") as f:
+                                    self.tokenizer = pickle.load(f)
+                            else:
+                                raise TransformerConnectionError()
+
+            except Exception as e:
+                raise RuntimeError(
+                    "Exception: {} Fail to load the model from {}! ".format(
+                        e, model_arg
+                    )
+                )
+
+        self.infer_dataloader = None
+        self.opt.eval_batch_size = kwargs.pop("eval_batch_size", 128)
+
+        self.opt.initializer = self.opt.initializer
+
+        if self.cal_perplexity:
+            try:
+                self.MLM, self.MLM_tokenizer = get_mlm_and_tokenizer(self, self.opt)
+            except Exception as e:
+                self.MLM, self.MLM_tokenizer = None, None
+
+        self.to(self.opt.device)
+
+    def to(self, device=None):
+        self.opt.device = device
+        self.model.to(device)
+        if hasattr(self, "MLM"):
+            self.MLM.to(self.opt.device)
+
+    def cpu(self):
+        self.opt.device = "cpu"
+        self.model.to("cpu")
+        if hasattr(self, "MLM"):
+            self.MLM.to("cpu")
+
+    def cuda(self, device="cuda:0"):
+        self.opt.device = device
+        self.model.to(device)
+        if hasattr(self, "MLM"):
+            self.MLM.to(device)
+
+    def _log_write_args(self):
+        n_trainable_params, n_nontrainable_params = 0, 0
+        for p in self.model.parameters():
+            n_params = torch.prod(torch.tensor(p.shape))
+            if p.requires_grad:
+                n_trainable_params += n_params
+            else:
+                n_nontrainable_params += n_params
+        print(
+            "n_trainable_params: {0}, n_nontrainable_params: {1}".format(
+                n_trainable_params, n_nontrainable_params
+            )
+        )
+        for arg in vars(self.opt):
+            if getattr(self.opt, arg) is not None:
+                print(">>> {0}: {1}".format(arg, getattr(self.opt, arg)))
+
+    def batch_infer(
+        self,
+        target_file=None,
+        print_result=True,
+        save_result=False,
+        ignore_error=True,
+        defense: str = None,
+    ):
+        save_path = os.path.join(os.getcwd(), "tad_text_classification.result.json")
+
+        target_file = detect_infer_dataset(target_file, task="text_defense")
+        if not target_file:
+            raise FileNotFoundError("Can not find inference datasets!")
+
+        if hasattr(BERTTADModelList, self.opt.model.__name__):
+            dataset = BERTTADDataset(tokenizer=self.tokenizer, opt=self.opt)
+
+        dataset.prepare_infer_dataset(target_file, ignore_error=ignore_error)
+        self.infer_dataloader = DataLoader(
+            dataset=dataset,
+            batch_size=self.opt.eval_batch_size,
+            pin_memory=True,
+            shuffle=False,
+        )
+        return self._infer(
+            save_path=save_path if save_result else None,
+            print_result=print_result,
+            defense=defense,
+        )
+
+    def infer(
+        self,
+        text: str = None,
+        print_result=True,
+        ignore_error=True,
+        defense: str = None,
+    ):
+        if hasattr(BERTTADModelList, self.opt.model.__name__):
+            dataset = BERTTADDataset(tokenizer=self.tokenizer, opt=self.opt)
+
+        if text:
+            dataset.prepare_infer_sample(text, ignore_error=ignore_error)
+        else:
+            raise RuntimeError("Please specify your datasets path!")
+        self.infer_dataloader = DataLoader(
+            dataset=dataset, batch_size=self.opt.eval_batch_size, shuffle=False
+        )
+        return self._infer(print_result=print_result, defense=defense)[0]
+
+    def _infer(self, save_path=None, print_result=True, defense=None):
+        _params = filter(lambda p: p.requires_grad, self.model.parameters())
+
+        correct = {True: "Correct", False: "Wrong"}
+        results = []
+
+        with torch.no_grad():
+            self.model.eval()
+            n_correct = 0
+            n_labeled = 0
+
+            n_advdet_correct = 0
+            n_advdet_labeled = 0
+            if len(self.infer_dataloader.dataset) >= 100:
+                it = tqdm.tqdm(self.infer_dataloader, postfix="inferring...")
+            else:
+                it = self.infer_dataloader
+            for _, sample in enumerate(it):
+                inputs = [
+                    sample[col].to(self.opt.device) for col in self.opt.inputs_cols
+                ]
+                outputs = self.model(inputs)
+                logits, advdet_logits, adv_tr_logits = (
+                    outputs["sent_logits"],
+                    outputs["advdet_logits"],
+                    outputs["adv_tr_logits"],
+                )
+                probs, advdet_probs, adv_tr_probs = (
+                    torch.softmax(logits, dim=-1),
+                    torch.softmax(advdet_logits, dim=-1),
+                    torch.softmax(adv_tr_logits, dim=-1),
+                )
+
+                for i, (prob, advdet_prob, adv_tr_prob) in enumerate(
+                    zip(probs, advdet_probs, adv_tr_probs)
+                ):
+                    text_raw = sample["text_raw"][i]
+
+                    pred_label = int(prob.argmax(axis=-1))
+                    pred_is_adv_label = int(advdet_prob.argmax(axis=-1))
+                    pred_adv_tr_label = int(adv_tr_prob.argmax(axis=-1))
+                    ref_label = (
+                        int(sample["label"][i])
+                        if int(sample["label"][i]) in self.opt.index_to_label
+                        else ""
+                    )
+                    ref_is_adv_label = (
+                        int(sample["is_adv"][i])
+                        if int(sample["is_adv"][i]) in self.opt.index_to_is_adv
+                        else ""
+                    )
+                    ref_adv_tr_label = (
+                        int(sample["adv_train_label"][i])
+                        if int(sample["adv_train_label"][i])
+                        in self.opt.index_to_adv_train_label
+                        else ""
+                    )
+
+                    if self.cal_perplexity:
+                        ids = self.MLM_tokenizer(text_raw, return_tensors="pt")
+                        ids["labels"] = ids["input_ids"].clone()
+                        ids = ids.to(self.opt.device)
+                        loss = self.MLM(**ids)["loss"]
+                        perplexity = float(torch.exp(loss / ids["input_ids"].size(1)))
+                    else:
+                        perplexity = "N.A."
+
+                    result = {
+                        "text": text_raw,
+                        "label": self.opt.index_to_label[pred_label],
+                        "probs": prob.cpu().numpy(),
+                        "confidence": float(max(prob)),
+                        "ref_label": self.opt.index_to_label[ref_label]
+                        if isinstance(ref_label, int)
+                        else ref_label,
+                        "ref_label_check": correct[pred_label == ref_label]
+                        if ref_label != -100
+                        else "",
+                        "is_fixed": False,
+                        "is_adv_label": self.opt.index_to_is_adv[pred_is_adv_label],
+                        "is_adv_probs": advdet_prob.cpu().numpy(),
+                        "is_adv_confidence": float(max(advdet_prob)),
+                        "ref_is_adv_label": self.opt.index_to_is_adv[ref_is_adv_label]
+                        if isinstance(ref_is_adv_label, int)
+                        else ref_is_adv_label,
+                        "ref_is_adv_check": correct[
+                            pred_is_adv_label == ref_is_adv_label
+                        ]
+                        if ref_is_adv_label != -100
+                        and isinstance(ref_is_adv_label, int)
+                        else "",
+                        "pred_adv_tr_label": self.opt.index_to_label[pred_adv_tr_label],
+                        "ref_adv_tr_label": self.opt.index_to_label[ref_adv_tr_label],
+                        "perplexity": perplexity,
+                    }
+                    if defense:
+                        try:
+                            if not hasattr(self, "sent_attacker"):
+                                self.sent_attacker = init_attacker(
+                                    self, defense.lower()
+                                )
+                            if result["is_adv_label"] == "1":
+                                res = self.sent_attacker.attacker.simple_attack(
+                                    text_raw, int(result["label"])
+                                )
+                                new_infer_res = self.infer(
+                                    res.perturbed_result.attacked_text.text,
+                                    print_result=False,
+                                )
+                                result["perturbed_label"] = result["label"]
+                                result["label"] = new_infer_res["label"]
+                                result["probs"] = new_infer_res["probs"]
+                                result["ref_label_check"] = (
+                                    correct[int(result["label"]) == ref_label]
+                                    if ref_label != -100
+                                    else ""
+                                )
+                                result[
+                                    "restored_text"
+                                ] = res.perturbed_result.attacked_text.text
+                                result["is_fixed"] = True
+                            else:
+                                result["restored_text"] = ""
+                                result["is_fixed"] = False
+
+                        except Exception as e:
+                            print(
+                                "Error:{}, try install TextAttack and tensorflow_text after 10 seconds...".format(
+                                    e
+                                )
+                            )
+                            time.sleep(10)
+                            raise RuntimeError("Installation done, please run again...")
+
+                    if ref_label != -100:
+                        n_labeled += 1
+
+                        if result["label"] == result["ref_label"]:
+                            n_correct += 1
+
+                    if ref_is_adv_label != -100:
+                        n_advdet_labeled += 1
+                        if ref_is_adv_label == pred_is_adv_label:
+                            n_advdet_correct += 1
+
+                    results.append(result)
+
+        try:
+            if print_result:
+                for ex_id, result in enumerate(results):
+                    text_printing = result["text"][:]
+                    text_info = ""
+                    if result["label"] != "-100":
+                        if not result["ref_label"]:
+                            text_info += " -> <CLS:{}(ref:{} confidence:{})>".format(
+                                result["label"],
+                                result["ref_label"],
+                                result["confidence"],
+                            )
+                        elif result["label"] == result["ref_label"]:
+                            text_info += colored(
+                                " -> <CLS:{}(ref:{} confidence:{})>".format(
+                                    result["label"],
+                                    result["ref_label"],
+                                    result["confidence"],
+                                ),
+                                "green",
+                            )
+                        else:
+                            text_info += colored(
+                                " -> <CLS:{}(ref:{} confidence:{})>".format(
+                                    result["label"],
+                                    result["ref_label"],
+                                    result["confidence"],
+                                ),
+                                "red",
+                            )
+
+                    # AdvDet
+                    if result["is_adv_label"] != "-100":
+                        if not result["ref_is_adv_label"]:
+                            text_info += " -> <AdvDet:{}(ref:{} confidence:{})>".format(
+                                result["is_adv_label"],
+                                result["ref_is_adv_check"],
+                                result["is_adv_confidence"],
+                            )
+                        elif result["is_adv_label"] == result["ref_is_adv_label"]:
+                            text_info += colored(
+                                " -> <AdvDet:{}(ref:{} confidence:{})>".format(
+                                    result["is_adv_label"],
+                                    result["ref_is_adv_label"],
+                                    result["is_adv_confidence"],
+                                ),
+                                "green",
+                            )
+                        else:
+                            text_info += colored(
+                                " -> <AdvDet:{}(ref:{} confidence:{})>".format(
+                                    result["is_adv_label"],
+                                    result["ref_is_adv_label"],
+                                    result["is_adv_confidence"],
+                                ),
+                                "red",
+                            )
+                    text_printing += text_info
+                    if self.cal_perplexity:
+                        text_printing += colored(
+                            " --> <perplexity:{}>".format(result["perplexity"]),
+                            "yellow",
+                        )
+                    print("Example {}: {}".format(ex_id, text_printing))
+            if save_path:
+                with open(save_path, "w", encoding="utf8") as fout:
+                    json.dump(str(results), fout, ensure_ascii=False)
+                    print("inference result saved in: {}".format(save_path))
+        except Exception as e:
+            print("Can not save result: {}, Exception: {}".format(text_raw, e))
+
+        if len(results) > 1:
+            print(
+                "CLS Acc:{}%".format(100 * n_correct / n_labeled if n_labeled else "")
+            )
+            print(
+                "AdvDet Acc:{}%".format(
+                    100 * n_advdet_correct / n_advdet_labeled
+                    if n_advdet_labeled
+                    else ""
+                )
+            )
+
+        return results
+
+    def clear_input_samples(self):
+        self.dataset.all_data = []

anonymous_demo/functional/__init__.py

@@ -0,0 +1,3 @@
+from anonymous_demo.functional.checkpoint.checkpoint_manager import TADCheckpointManager
+
+from anonymous_demo.functional.config import TADConfigManager

anonymous_demo/functional/checkpoint/__init__.py

@@ -0,0 +1 @@
+from .checkpoint_manager import TADCheckpointManager

anonymous_demo/functional/checkpoint/checkpoint_manager.py

@@ -0,0 +1,19 @@
+import os
+from findfile import find_file
+
+from anonymous_demo.core.tad.prediction.tad_classifier import TADTextClassifier
+from anonymous_demo.utils.demo_utils import retry
+
+
+class CheckpointManager:
+    pass
+
+
+class TADCheckpointManager(CheckpointManager):
+    @staticmethod
+    @retry
+    def get_tad_text_classifier(checkpoint: str = None, eval_batch_size=128, **kwargs):
+        tad_text_classifier = TADTextClassifier(
+            checkpoint, eval_batch_size=eval_batch_size, **kwargs
+        )
+        return tad_text_classifier

anonymous_demo/functional/config/__init__.py

@@ -0,0 +1 @@
+from .tad_config_manager import TADConfigManager

anonymous_demo/functional/config/config_manager.py

@@ -0,0 +1,64 @@
+from argparse import Namespace
+
+import torch
+
+one_shot_messages = set()
+
+
+def config_check(args):
+    pass
+
+
+class ConfigManager(Namespace):
+    def __init__(self, args=None, **kwargs):
+        """
+        The ConfigManager is a subclass of argparse.Namespace and based on parameter dict and count the call-frequency of each parameter
+        :param args: A parameter dict
+        :param kwargs: Same param as Namespce
+        """
+        if not args:
+            args = {}
+        super().__init__(**kwargs)
+
+        if isinstance(args, Namespace):
+            self.args = vars(args)
+            self.args_call_count = {arg: 0 for arg in vars(args)}
+        else:
+            self.args = args
+            self.args_call_count = {arg: 0 for arg in args}
+
+    def __getattribute__(self, arg_name):
+        if arg_name == "args" or arg_name == "args_call_count":
+            return super().__getattribute__(arg_name)
+        try:
+            value = super().__getattribute__("args")[arg_name]
+            args_call_count = super().__getattribute__("args_call_count")
+            args_call_count[arg_name] += 1
+            super().__setattr__("args_call_count", args_call_count)
+            return value
+
+        except Exception as e:
+            return super().__getattribute__(arg_name)
+
+    def __setattr__(self, arg_name, value):
+        if arg_name == "args" or arg_name == "args_call_count":
+            super().__setattr__(arg_name, value)
+            return
+        try:
+            args = super().__getattribute__("args")
+            args[arg_name] = value
+            super().__setattr__("args", args)
+            args_call_count = super().__getattribute__("args_call_count")
+
+            if arg_name in args_call_count:
+                # args_call_count[arg_name] += 1
+                super().__setattr__("args_call_count", args_call_count)
+
+            else:
+                args_call_count[arg_name] = 0
+                super().__setattr__("args_call_count", args_call_count)
+
+        except Exception as e:
+            super().__setattr__(arg_name, value)
+
+        config_check(args)

anonymous_demo/functional/config/tad_config_manager.py

@@ -0,0 +1,229 @@
+import copy
+
+from anonymous_demo.functional.config.config_manager import ConfigManager
+from anonymous_demo.core.tad.classic.__bert__.models import TADBERT
+
+_tad_config_template = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "patience": 99999,
+    "pretrained_bert": "microsoft/mdeberta-v3-base",
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "max_seq_len": 80,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1,
+    "use_amp": False,
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+_tad_config_base = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "pretrained_bert": "microsoft/deberta-v3-base",
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "max_seq_len": 80,
+    "patience": 99999,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+_tad_config_english = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "patience": 99999,
+    "pretrained_bert": "microsoft/deberta-v3-base",
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "max_seq_len": 80,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+_tad_config_multilingual = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "patience": 99999,
+    "pretrained_bert": "microsoft/mdeberta-v3-base",
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "max_seq_len": 80,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+_tad_config_chinese = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "patience": 99999,
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "pretrained_bert": "bert-base-chinese",
+    "max_seq_len": 80,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+
+class TADConfigManager(ConfigManager):
+    def __init__(self, args, **kwargs):
+        """
+        Available Params:  {'model': BERT,
+                            'optimizer': "adamw",
+                            'learning_rate': 0.00002,
+                            'pretrained_bert': "roberta-base",
+                            'cache_dataset': True,
+                            'warmup_step': -1,
+                            'show_metric': False,
+                            'max_seq_len': 80,
+                            'patience': 99999,
+                            'dropout': 0,
+                            'l2reg': 0.000001,
+                            'num_epoch': 10,
+                            'batch_size': 16,
+                            'initializer': 'xavier_uniform_',
+                            'seed': {52, 25}
+                            'embed_dim': 768,
+                            'hidden_dim': 768,
+                            'polarities_dim': 3,
+                            'log_step': 10,
+                            'evaluate_begin': 0,
+                            'cross_validate_fold': -1 # split train and test datasets into 5 folds and repeat 3 training
+                            }
+        :param args:
+        :param kwargs:
+        """
+        super().__init__(args, **kwargs)
+
+    @staticmethod
+    def set_tad_config(configType: str, newitem: dict):
+        if isinstance(newitem, dict):
+            if configType == "template":
+                _tad_config_template.update(newitem)
+            elif configType == "base":
+                _tad_config_base.update(newitem)
+            elif configType == "english":
+                _tad_config_english.update(newitem)
+            elif configType == "chinese":
+                _tad_config_chinese.update(newitem)
+            elif configType == "multilingual":
+                _tad_config_multilingual.update(newitem)
+            elif configType == "glove":
+                _tad_config_glove.update(newitem)
+            else:
+                raise ValueError(
+                    "Wrong value of config type supplied, please use one from following type: template, base, english, chinese, multilingual, glove"
+                )
+        else:
+            raise TypeError(
+                "Wrong type of new config item supplied, please use dict e.g.{'NewConfig': NewValue}"
+            )
+
+    @staticmethod
+    def set_tad_config_template(newitem):
+        TADConfigManager.set_tad_config("template", newitem)
+
+    @staticmethod
+    def set_tad_config_base(newitem):
+        TADConfigManager.set_tad_config("base", newitem)
+
+    @staticmethod
+    def set_tad_config_english(newitem):
+        TADConfigManager.set_tad_config("english", newitem)
+
+    @staticmethod
+    def set_tad_config_chinese(newitem):
+        TADConfigManager.set_tad_config("chinese", newitem)
+
+    @staticmethod
+    def set_tad_config_multilingual(newitem):
+        TADConfigManager.set_tad_config("multilingual", newitem)
+
+    @staticmethod
+    def set_tad_config_glove(newitem):
+        TADConfigManager.set_tad_config("glove", newitem)
+
+    @staticmethod
+    def get_tad_config_template() -> ConfigManager:
+        _tad_config_template.update(_tad_config_template)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_base() -> ConfigManager:
+        _tad_config_template.update(_tad_config_base)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_english() -> ConfigManager:
+        _tad_config_template.update(_tad_config_english)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_chinese() -> ConfigManager:
+        _tad_config_template.update(_tad_config_chinese)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_multilingual() -> ConfigManager:
+        _tad_config_template.update(_tad_config_multilingual)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_glove() -> ConfigManager:
+        _tad_config_template.update(_tad_config_glove)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))

anonymous_demo/functional/dataset/__init__.py

@@ -0,0 +1 @@
+from anonymous_demo.functional.dataset.dataset_manager import detect_infer_dataset

anonymous_demo/functional/dataset/dataset_manager.py

@@ -0,0 +1,45 @@
+import os
+from findfile import find_files, find_dir
+
+filter_key_words = [
+    ".py",
+    ".md",
+    "readme",
+    "log",
+    "result",
+    "zip",
+    ".state_dict",
+    ".model",
+    ".png",
+    "acc_",
+    "f1_",
+    ".backup",
+    ".bak",
+]
+
+
+def detect_infer_dataset(dataset_path, task="apc"):
+    dataset_file = []
+    if isinstance(dataset_path, str) and os.path.isfile(dataset_path):
+        dataset_file.append(dataset_path)
+        return dataset_file
+
+    for d in dataset_path:
+        if not os.path.exists(d):
+            search_path = find_dir(
+                os.getcwd(),
+                [d, task, "dataset"],
+                exclude_key=filter_key_words,
+                disable_alert=False,
+            )
+            dataset_file += find_files(
+                search_path,
+                [".inference", d],
+                exclude_key=["train."] + filter_key_words,
+            )
+        else:
+            dataset_file += find_files(
+                d, [".inference", task], exclude_key=["train."] + filter_key_words
+            )
+
+    return dataset_file

anonymous_demo/network/lcf_pooler.py

@@ -0,0 +1,28 @@
+import numpy
+import torch
+import torch.nn as nn
+
+
+class LCF_Pooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = nn.Tanh()
+
+    def forward(self, hidden_states, lcf_vec):
+        device = hidden_states.device
+        lcf_vec = lcf_vec.detach().cpu().numpy()
+
+        pooled_output = numpy.zeros(
+            (hidden_states.shape[0], hidden_states.shape[2]), dtype=numpy.float32
+        )
+        hidden_states = hidden_states.detach().cpu().numpy()
+        for i, vec in enumerate(lcf_vec):
+            lcf_ids = [j for j in range(len(vec)) if sum(vec[j] - 1.0) == 0]
+            pooled_output[i] = hidden_states[i][lcf_ids[len(lcf_ids) // 2]]
+
+        pooled_output = torch.Tensor(pooled_output).to(device)
+        pooled_output = self.dense(pooled_output)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output

anonymous_demo/network/lsa.py

@@ -0,0 +1,73 @@
+import torch
+from anonymous_demo.network.sa_encoder import Encoder
+from torch import nn
+
+
+class LSA(nn.Module):
+    def __init__(self, bert, opt):
+        super(LSA, self).__init__()
+        self.opt = opt
+
+        self.encoder = Encoder(bert.config, opt)
+        self.encoder_left = Encoder(bert.config, opt)
+        self.encoder_right = Encoder(bert.config, opt)
+        self.linear_window_3h = nn.Linear(opt.embed_dim * 3, opt.embed_dim)
+        self.linear_window_2h = nn.Linear(opt.embed_dim * 2, opt.embed_dim)
+        self.eta1 = nn.Parameter(torch.tensor(self.opt.eta, dtype=torch.float))
+        self.eta2 = nn.Parameter(torch.tensor(self.opt.eta, dtype=torch.float))
+
+    def forward(
+        self,
+        global_context_features,
+        spc_mask_vec,
+        lcf_matrix,
+        left_lcf_matrix,
+        right_lcf_matrix,
+    ):
+        masked_global_context_features = torch.mul(
+            spc_mask_vec, global_context_features
+        )
+
+        # # --------------------------------------------------- #
+        lcf_features = torch.mul(global_context_features, lcf_matrix)
+        lcf_features = self.encoder(lcf_features)
+        # # --------------------------------------------------- #
+        left_lcf_features = torch.mul(masked_global_context_features, left_lcf_matrix)
+        left_lcf_features = self.encoder_left(left_lcf_features)
+        # # --------------------------------------------------- #
+        right_lcf_features = torch.mul(masked_global_context_features, right_lcf_matrix)
+        right_lcf_features = self.encoder_right(right_lcf_features)
+        # # --------------------------------------------------- #
+        if "lr" == self.opt.window or "rl" == self.opt.window:
+            if self.eta1 <= 0 and self.opt.eta != -1:
+                torch.nn.init.uniform_(self.eta1)
+                print("reset eta1 to: {}".format(self.eta1.item()))
+            if self.eta2 <= 0 and self.opt.eta != -1:
+                torch.nn.init.uniform_(self.eta2)
+                print("reset eta2 to: {}".format(self.eta2.item()))
+            if self.opt.eta >= 0:
+                cat_features = torch.cat(
+                    (
+                        lcf_features,
+                        self.eta1 * left_lcf_features,
+                        self.eta2 * right_lcf_features,
+                    ),
+                    -1,
+                )
+            else:
+                cat_features = torch.cat(
+                    (lcf_features, left_lcf_features, right_lcf_features), -1
+                )
+            sent_out = self.linear_window_3h(cat_features)
+        elif "l" == self.opt.window:
+            sent_out = self.linear_window_2h(
+                torch.cat((lcf_features, self.eta1 * left_lcf_features), -1)
+            )
+        elif "r" == self.opt.window:
+            sent_out = self.linear_window_2h(
+                torch.cat((lcf_features, self.eta2 * right_lcf_features), -1)
+            )
+        else:
+            raise KeyError("Invalid parameter:", self.opt.window)
+
+        return sent_out

anonymous_demo/network/sa_encoder.py

@@ -0,0 +1,199 @@
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+
+class BertSelfAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(
+            config, "embedding_size"
+        ):
+            raise ValueError(
+                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({config.num_attention_heads})"
+            )
+
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        self.key = nn.Linear(config.hidden_size, self.all_head_size)
+        self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(
+            config.attention_probs_dropout_prob
+            if hasattr(config, "attention_probs_dropout_prob")
+            else 0
+        )
+        self.position_embedding_type = getattr(
+            config, "position_embedding_type", "absolute"
+        )
+        if (
+            self.position_embedding_type == "relative_key"
+            or self.position_embedding_type == "relative_key_query"
+        ):
+            self.max_position_embeddings = config.max_position_embeddings
+            self.distance_embedding = nn.Embedding(
+                2 * config.max_position_embeddings - 1, self.attention_head_size
+            )
+
+        self.is_decoder = config.is_decoder
+
+    def transpose_for_scores(self, x):
+        new_x_shape = x.size()[:-1] + (
+            self.num_attention_heads,
+            self.attention_head_size,
+        )
+        x = x.view(*new_x_shape)
+        return x.permute(0, 2, 1, 3)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+    ):
+        mixed_query_layer = self.query(hidden_states)
+
+        # If this is instantiated as a cross-attention module, the keys
+        # and values come from an encoder; the attention mask needs to be
+        # such that the encoder's padding tokens are not attended to.
+        is_cross_attention = encoder_hidden_states is not None
+
+        if is_cross_attention and past_key_value is not None:
+            # reuse k,v, cross_attentions
+            key_layer = past_key_value[0]
+            value_layer = past_key_value[1]
+            attention_mask = encoder_attention_mask
+        elif is_cross_attention:
+            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
+            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
+            attention_mask = encoder_attention_mask
+        elif past_key_value is not None:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+            key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
+            value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
+        else:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+
+        if self.is_decoder:
+            # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
+            # Further calls to cross_attention layer can then reuse all cross-attention
+            # key/value_states (first "if" case)
+            # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
+            # all previous decoder key/value_states. Further calls to uni-directional self-attention
+            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
+            # if encoder bi-directional self-attention `past_key_value` is always `None`
+            past_key_value = (key_layer, value_layer)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+
+        if (
+            self.position_embedding_type == "relative_key"
+            or self.position_embedding_type == "relative_key_query"
+        ):
+            seq_length = hidden_states.size()[1]
+            position_ids_l = torch.arange(
+                seq_length, dtype=torch.long, device=hidden_states.device
+            ).view(-1, 1)
+            position_ids_r = torch.arange(
+                seq_length, dtype=torch.long, device=hidden_states.device
+            ).view(1, -1)
+            distance = position_ids_l - position_ids_r
+            positional_embedding = self.distance_embedding(
+                distance + self.max_position_embeddings - 1
+            )
+            positional_embedding = positional_embedding.to(
+                dtype=query_layer.dtype
+            )  # fp16 compatibility
+
+            if self.position_embedding_type == "relative_key":
+                relative_position_scores = torch.einsum(
+                    "bhld,lrd->bhlr", query_layer, positional_embedding
+                )
+                attention_scores = attention_scores + relative_position_scores
+            elif self.position_embedding_type == "relative_key_query":
+                relative_position_scores_query = torch.einsum(
+                    "bhld,lrd->bhlr", query_layer, positional_embedding
+                )
+                relative_position_scores_key = torch.einsum(
+                    "bhrd,lrd->bhlr", key_layer, positional_embedding
+                )
+                attention_scores = (
+                    attention_scores
+                    + relative_position_scores_query
+                    + relative_position_scores_key
+                )
+
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.Softmax(dim=-1)(attention_scores)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs = attention_probs * head_mask
+
+        context_layer = torch.matmul(attention_probs, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(*new_context_layer_shape)
+
+        outputs = (
+            (context_layer, attention_probs) if output_attentions else (context_layer,)
+        )
+
+        if self.is_decoder:
+            outputs = outputs + (past_key_value,)
+        return outputs
+
+
+class Encoder(nn.Module):
+    def __init__(self, config, opt, layer_num=1):
+        super(Encoder, self).__init__()
+        self.opt = opt
+        self.config = config
+        self.encoder = nn.ModuleList(
+            [SelfAttention(config, opt) for _ in range(layer_num)]
+        )
+        self.tanh = torch.nn.Tanh()
+
+    def forward(self, x):
+        for i, enc in enumerate(self.encoder):
+            x = self.tanh(enc(x)[0])
+        return x
+
+
+class SelfAttention(nn.Module):
+    def __init__(self, config, opt):
+        super(SelfAttention, self).__init__()
+        self.opt = opt
+        self.config = config
+        self.SA = BertSelfAttention(config)
+
+    def forward(self, inputs):
+        zero_vec = np.zeros((inputs.size(0), 1, 1, self.opt.max_seq_len))
+        zero_tensor = torch.tensor(zero_vec).float().to(inputs.device)
+        SA_out = self.SA(inputs, zero_tensor)
+        return SA_out

anonymous_demo/utils/demo_utils.py

@@ -0,0 +1,247 @@
+import json
+import os
+import pickle
+import signal
+import threading
+import time
+import zipfile
+
+import gdown
+import numpy as np
+import requests
+import torch
+import tqdm
+from autocuda import auto_cuda, auto_cuda_name
+from findfile import find_files, find_cwd_file, find_file
+from termcolor import colored
+from functools import wraps
+
+from update_checker import parse_version
+
+from anonymous_demo import __version__
+
+
+def save_args(config, save_path):
+    f = open(os.path.join(save_path), mode="w", encoding="utf8")
+    for arg in config.args:
+        if config.args_call_count[arg]:
+            f.write("{}: {}\n".format(arg, config.args[arg]))
+    f.close()
+
+
+def print_args(config, logger=None, mode=0):
+    args = [key for key in sorted(config.args.keys())]
+    for arg in args:
+        if logger:
+            logger.info(
+                "{0}:{1}\t-->\tCalling Count:{2}".format(
+                    arg, config.args[arg], config.args_call_count[arg]
+                )
+            )
+        else:
+            print(
+                "{0}:{1}\t-->\tCalling Count:{2}".format(
+                    arg, config.args[arg], config.args_call_count[arg]
+                )
+            )
+
+
+def check_and_fix_labels(label_set: set, label_name, all_data, opt):
+    if "-100" in label_set:
+        label_to_index = {
+            origin_label: int(idx) - 1 if origin_label != "-100" else -100
+            for origin_label, idx in zip(sorted(label_set), range(len(label_set)))
+        }
+        index_to_label = {
+            int(idx) - 1 if origin_label != "-100" else -100: origin_label
+            for origin_label, idx in zip(sorted(label_set), range(len(label_set)))
+        }
+    else:
+        label_to_index = {
+            origin_label: int(idx)
+            for origin_label, idx in zip(sorted(label_set), range(len(label_set)))
+        }
+        index_to_label = {
+            int(idx): origin_label
+            for origin_label, idx in zip(sorted(label_set), range(len(label_set)))
+        }
+    if "index_to_label" not in opt.args:
+        opt.index_to_label = index_to_label
+        opt.label_to_index = label_to_index
+
+    if opt.index_to_label != index_to_label:
+        opt.index_to_label.update(index_to_label)
+        opt.label_to_index.update(label_to_index)
+    num_label = {l: 0 for l in label_set}
+    num_label["Sum"] = len(all_data)
+    for item in all_data:
+        try:
+            num_label[item[label_name]] += 1
+            item[label_name] = label_to_index[item[label_name]]
+        except Exception as e:
+            # print(e)
+            num_label[item.polarity] += 1
+            item.polarity = label_to_index[item.polarity]
+    print("Dataset Label Details: {}".format(num_label))
+
+
+def check_and_fix_IOB_labels(label_map, opt):
+    index_to_IOB_label = {
+        int(label_map[origin_label]): origin_label for origin_label in label_map
+    }
+    opt.index_to_IOB_label = index_to_IOB_label
+
+
+def get_device(auto_device):
+    if isinstance(auto_device, str) and auto_device == "allcuda":
+        device = "cuda"
+    elif isinstance(auto_device, str):
+        device = auto_device
+    elif isinstance(auto_device, bool):
+        device = auto_cuda() if auto_device else "cpu"
+    else:
+        device = auto_cuda()
+        try:
+            torch.device(device)
+        except RuntimeError as e:
+            print(
+                colored("Device assignment error: {}, redirect to CPU".format(e), "red")
+            )
+            device = "cpu"
+    device_name = auto_cuda_name()
+    return device, device_name
+
+
+def _load_word_vec(path, word2idx=None, embed_dim=300):
+    fin = open(path, "r", encoding="utf-8", newline="\n", errors="ignore")
+    word_vec = {}
+    for line in tqdm.tqdm(fin.readlines(), postfix="Loading embedding file..."):
+        tokens = line.rstrip().split()
+        word, vec = " ".join(tokens[:-embed_dim]), tokens[-embed_dim:]
+        if word in word2idx.keys():
+            word_vec[word] = np.asarray(vec, dtype="float32")
+    return word_vec
+
+
+def build_embedding_matrix(word2idx, embed_dim, dat_fname, opt):
+    if not os.path.exists("run"):
+        os.makedirs("run")
+    embed_matrix_path = "run/{}".format(os.path.join(opt.dataset_name, dat_fname))
+    if os.path.exists(embed_matrix_path):
+        print(
+            colored(
+                "Loading cached embedding_matrix from {} (Please remove all cached files if there is any problem!)".format(
+                    embed_matrix_path
+                ),
+                "green",
+            )
+        )
+        embedding_matrix = pickle.load(open(embed_matrix_path, "rb"))
+    else:
+        glove_path = prepare_glove840_embedding(embed_matrix_path)
+        embedding_matrix = np.zeros((len(word2idx) + 2, embed_dim))
+
+        word_vec = _load_word_vec(glove_path, word2idx=word2idx, embed_dim=embed_dim)
+
+        for word, i in tqdm.tqdm(
+            word2idx.items(),
+            postfix=colored("Building embedding_matrix {}".format(dat_fname), "yellow"),
+        ):
+            vec = word_vec.get(word)
+            if vec is not None:
+                embedding_matrix[i] = vec
+        pickle.dump(embedding_matrix, open(embed_matrix_path, "wb"))
+    return embedding_matrix
+
+
+def pad_and_truncate(
+    sequence, maxlen, dtype="int64", padding="post", truncating="post", value=0
+):
+    x = (np.ones(maxlen) * value).astype(dtype)
+    if truncating == "pre":
+        trunc = sequence[-maxlen:]
+    else:
+        trunc = sequence[:maxlen]
+    trunc = np.asarray(trunc, dtype=dtype)
+    if padding == "post":
+        x[: len(trunc)] = trunc
+    else:
+        x[-len(trunc) :] = trunc
+    return x
+
+
+class TransformerConnectionError(ValueError):
+    def __init__(self):
+        pass
+
+
+def retry(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        count = 5
+        while count:
+            try:
+                return f(*args, **kwargs)
+            except (
+                TransformerConnectionError,
+                requests.exceptions.RequestException,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.HTTPError,
+                requests.exceptions.ConnectTimeout,
+                requests.exceptions.ProxyError,
+                requests.exceptions.SSLError,
+                requests.exceptions.BaseHTTPError,
+            ) as e:
+                print(colored("Training Exception: {}, will retry later".format(e)))
+                time.sleep(60)
+                count -= 1
+
+    return decorated
+
+
+def save_json(dic, save_path):
+    if isinstance(dic, str):
+        dic = eval(dic)
+    with open(save_path, "w", encoding="utf-8") as f:
+        # f.write(str(dict))
+        str_ = json.dumps(dic, ensure_ascii=False)
+        f.write(str_)
+
+
+def load_json(save_path):
+    with open(save_path, "r", encoding="utf-8") as f:
+        data = f.readline().strip()
+        print(type(data), data)
+        dic = json.loads(data)
+    return dic
+
+
+def init_optimizer(optimizer):
+    optimizers = {
+        "adadelta": torch.optim.Adadelta,  # default lr=1.0
+        "adagrad": torch.optim.Adagrad,  # default lr=0.01
+        "adam": torch.optim.Adam,  # default lr=0.001
+        "adamax": torch.optim.Adamax,  # default lr=0.002
+        "asgd": torch.optim.ASGD,  # default lr=0.01
+        "rmsprop": torch.optim.RMSprop,  # default lr=0.01
+        "sgd": torch.optim.SGD,
+        "adamw": torch.optim.AdamW,
+        torch.optim.Adadelta: torch.optim.Adadelta,  # default lr=1.0
+        torch.optim.Adagrad: torch.optim.Adagrad,  # default lr=0.01
+        torch.optim.Adam: torch.optim.Adam,  # default lr=0.001
+        torch.optim.Adamax: torch.optim.Adamax,  # default lr=0.002
+        torch.optim.ASGD: torch.optim.ASGD,  # default lr=0.01
+        torch.optim.RMSprop: torch.optim.RMSprop,  # default lr=0.01
+        torch.optim.SGD: torch.optim.SGD,
+        torch.optim.AdamW: torch.optim.AdamW,
+    }
+    if optimizer in optimizers:
+        return optimizers[optimizer]
+    elif hasattr(torch.optim, optimizer.__name__):
+        return optimizer
+    else:
+        raise KeyError(
+            "Unsupported optimizer: {}. Please use string or the optimizer objects in torch.optim as your optimizer".format(
+                optimizer
+            )
+        )

anonymous_demo/utils/logger.py

@@ -0,0 +1,38 @@
+import logging
+import os
+import sys
+import time
+
+import termcolor
+
+today = time.strftime("%Y%m%d %H%M%S", time.localtime(time.time()))
+
+
+def get_logger(log_path, log_name="", log_type="training_log"):
+    if not log_path:
+        log_dir = os.path.join(log_path, "logs")
+    else:
+        log_dir = os.path.join(".", "logs")
+
+    full_path = os.path.join(log_dir, log_name + "_" + today)
+    if not os.path.exists(full_path):
+        os.makedirs(full_path)
+    log_path = os.path.join(full_path, "{}.log".format(log_type))
+    logger = logging.getLogger(log_name)
+    if not logger.handlers:
+        formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
+
+        file_handler = logging.FileHandler(log_path, encoding="utf8")
+        file_handler.setFormatter(formatter)
+        file_handler.setLevel(logging.INFO)
+
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.formatter = formatter
+        console_handler.setLevel(logging.INFO)
+
+        logger.addHandler(file_handler)
+        logger.addHandler(console_handler)
+
+        logger.setLevel(logging.INFO)
+
+    return logger

checkpoints.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f77ae4a45785183900ee874cb318a16b0e2f173b31749a2555215aca93672f26
+size 2456834455

textattack/__init__.py

@@ -0,0 +1,39 @@
+"""Welcome to the API references for TextAttack!
+
+What is TextAttack?
+
+`TextAttack <https://github.com/QData/TextAttack>`__ is a Python framework for adversarial attacks, adversarial training, and data augmentation in NLP.
+
+TextAttack makes experimenting with the robustness of NLP models seamless, fast, and easy. It's also useful for NLP model training, adversarial training, and data augmentation.
+
+TextAttack provides components for common NLP tasks like sentence encoding, grammar-checking, and word replacement that can be used on their own.
+"""
+from .attack_args import AttackArgs, CommandLineAttackArgs
+from .augment_args import AugmenterArgs
+from .dataset_args import DatasetArgs
+from .model_args import ModelArgs
+from .training_args import TrainingArgs, CommandLineTrainingArgs
+from .attack import Attack
+from .attacker import Attacker
+from .trainer import Trainer
+from .metrics import Metric
+
+from . import (
+    attack_recipes,
+    attack_results,
+    augmentation,
+    commands,
+    constraints,
+    datasets,
+    goal_function_results,
+    goal_functions,
+    loggers,
+    metrics,
+    models,
+    search_methods,
+    shared,
+    transformations,
+)
+
+
+name = "textattack"

textattack/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+if __name__ == "__main__":
+    import textattack
+
+    textattack.commands.textattack_cli.main()

textattack/attack.py

@@ -0,0 +1,492 @@
+"""
+Attack Class
+============
+"""
+
+from collections import OrderedDict
+from typing import List, Union
+
+import lru
+import torch
+
+import textattack
+from textattack.attack_results import (
+    FailedAttackResult,
+    MaximizedAttackResult,
+    SkippedAttackResult,
+    SuccessfulAttackResult,
+)
+from textattack.constraints import Constraint, PreTransformationConstraint
+from textattack.goal_function_results import GoalFunctionResultStatus
+from textattack.goal_functions import GoalFunction
+from textattack.models.wrappers import ModelWrapper
+from textattack.search_methods import SearchMethod
+from textattack.shared import AttackedText, utils
+from textattack.transformations import CompositeTransformation, Transformation
+
+
+class Attack:
+    """An attack generates adversarial examples on text.
+
+    An attack is comprised of a goal function, constraints, transformation, and a search method. Use :meth:`attack` method to attack one sample at a time.
+
+    Args:
+        goal_function (:class:`~textattack.goal_functions.GoalFunction`):
+            A function for determining how well a perturbation is doing at achieving the attack's goal.
+        constraints (list of :class:`~textattack.constraints.Constraint` or :class:`~textattack.constraints.PreTransformationConstraint`):
+            A list of constraints to add to the attack, defining which perturbations are valid.
+        transformation (:class:`~textattack.transformations.Transformation`):
+            The transformation applied at each step of the attack.
+        search_method (:class:`~textattack.search_methods.SearchMethod`):
+            The method for exploring the search space of possible perturbations
+        transformation_cache_size (:obj:`int`, `optional`, defaults to :obj:`2**15`):
+            The number of items to keep in the transformations cache
+        constraint_cache_size (:obj:`int`, `optional`, defaults to :obj:`2**15`):
+            The number of items to keep in the constraints cache
+
+    Example::
+
+        >>> import textattack
+        >>> import transformers
+
+        >>> # Load model, tokenizer, and model_wrapper
+        >>> model = transformers.AutoModelForSequenceClassification.from_pretrained("textattack/bert-base-uncased-imdb")
+        >>> tokenizer = transformers.AutoTokenizer.from_pretrained("textattack/bert-base-uncased-imdb")
+        >>> model_wrapper = textattack.models.wrappers.HuggingFaceModelWrapper(model, tokenizer)
+
+        >>> # Construct our four components for `Attack`
+        >>> from textattack.constraints.pre_transformation import RepeatModification, StopwordModification
+        >>> from textattack.constraints.semantics import WordEmbeddingDistance
+
+        >>> goal_function = textattack.goal_functions.UntargetedClassification(model_wrapper)
+        >>> constraints = [
+        ...     RepeatModification(),
+        ...     StopwordModification()
+        ...     WordEmbeddingDistance(min_cos_sim=0.9)
+        ... ]
+        >>> transformation = WordSwapEmbedding(max_candidates=50)
+        >>> search_method = GreedyWordSwapWIR(wir_method="delete")
+
+        >>> # Construct the actual attack
+        >>> attack = Attack(goal_function, constraints, transformation, search_method)
+
+        >>> input_text = "I really enjoyed the new movie that came out last month."
+        >>> label = 1 #Positive
+        >>> attack_result = attack.attack(input_text, label)
+    """
+
+    def __init__(
+        self,
+        goal_function: GoalFunction,
+        constraints: List[Union[Constraint, PreTransformationConstraint]],
+        transformation: Transformation,
+        search_method: SearchMethod,
+        transformation_cache_size=2**15,
+        constraint_cache_size=2**15,
+    ):
+        """Initialize an attack object.
+
+        Attacks can be run multiple times.
+        """
+        assert isinstance(
+            goal_function, GoalFunction
+        ), f"`goal_function` must be of type `textattack.goal_functions.GoalFunction`, but got type `{type(goal_function)}`."
+        assert isinstance(
+            constraints, list
+        ), "`constraints` must be a list of `textattack.constraints.Constraint` or `textattack.constraints.PreTransformationConstraint`."
+        for c in constraints:
+            assert isinstance(
+                c, (Constraint, PreTransformationConstraint)
+            ), "`constraints` must be a list of `textattack.constraints.Constraint` or `textattack.constraints.PreTransformationConstraint`."
+        assert isinstance(
+            transformation, Transformation
+        ), f"`transformation` must be of type `textattack.transformations.Transformation`, but got type `{type(transformation)}`."
+        assert isinstance(
+            search_method, SearchMethod
+        ), f"`search_method` must be of type `textattack.search_methods.SearchMethod`, but got type `{type(search_method)}`."
+
+        self.goal_function = goal_function
+        self.search_method = search_method
+        self.transformation = transformation
+        self.is_black_box = (
+            getattr(transformation, "is_black_box", True) and search_method.is_black_box
+        )
+
+        if not self.search_method.check_transformation_compatibility(
+            self.transformation
+        ):
+            raise ValueError(
+                f"SearchMethod {self.search_method} incompatible with transformation {self.transformation}"
+            )
+
+        self.constraints = []
+        self.pre_transformation_constraints = []
+        for constraint in constraints:
+            if isinstance(
+                constraint,
+                textattack.constraints.PreTransformationConstraint,
+            ):
+                self.pre_transformation_constraints.append(constraint)
+            else:
+                self.constraints.append(constraint)
+
+        # Check if we can use transformation cache for our transformation.
+        if not self.transformation.deterministic:
+            self.use_transformation_cache = False
+        elif isinstance(self.transformation, CompositeTransformation):
+            self.use_transformation_cache = True
+            for t in self.transformation.transformations:
+                if not t.deterministic:
+                    self.use_transformation_cache = False
+                    break
+        else:
+            self.use_transformation_cache = True
+        self.transformation_cache_size = transformation_cache_size
+        self.transformation_cache = lru.LRU(transformation_cache_size)
+
+        self.constraint_cache_size = constraint_cache_size
+        self.constraints_cache = lru.LRU(constraint_cache_size)
+
+        # Give search method access to functions for getting transformations and evaluating them
+        self.search_method.get_transformations = self.get_transformations
+        # Give search method access to self.goal_function for model query count, etc.
+        self.search_method.goal_function = self.goal_function
+        # The search method only needs access to the first argument. The second is only used
+        # by the attack class when checking whether to skip the sample
+        self.search_method.get_goal_results = self.goal_function.get_results
+
+        # Give search method access to get indices which need to be ordered / searched
+        self.search_method.get_indices_to_order = self.get_indices_to_order
+
+        self.search_method.filter_transformations = self.filter_transformations
+
+    def clear_cache(self, recursive=True):
+        self.constraints_cache.clear()
+        if self.use_transformation_cache:
+            self.transformation_cache.clear()
+        if recursive:
+            self.goal_function.clear_cache()
+            for constraint in self.constraints:
+                if hasattr(constraint, "clear_cache"):
+                    constraint.clear_cache()
+
+    def cpu_(self):
+        """Move any `torch.nn.Module` models that are part of Attack to CPU."""
+        visited = set()
+
+        def to_cpu(obj):
+            visited.add(id(obj))
+            if isinstance(obj, torch.nn.Module):
+                obj.cpu()
+            elif isinstance(
+                obj,
+                (
+                    Attack,
+                    GoalFunction,
+                    Transformation,
+                    SearchMethod,
+                    Constraint,
+                    PreTransformationConstraint,
+                    ModelWrapper,
+                ),
+            ):
+                for key in obj.__dict__:
+                    s_obj = obj.__dict__[key]
+                    if id(s_obj) not in visited:
+                        to_cpu(s_obj)
+            elif isinstance(obj, (list, tuple)):
+                for item in obj:
+                    if id(item) not in visited and isinstance(
+                        item, (Transformation, Constraint, PreTransformationConstraint)
+                    ):
+                        to_cpu(item)
+
+        to_cpu(self)
+
+    def cuda_(self):
+        """Move any `torch.nn.Module` models that are part of Attack to GPU."""
+        visited = set()
+
+        def to_cuda(obj):
+            visited.add(id(obj))
+            if isinstance(obj, torch.nn.Module):
+                obj.to(textattack.shared.utils.device)
+            elif isinstance(
+                obj,
+                (
+                    Attack,
+                    GoalFunction,
+                    Transformation,
+                    SearchMethod,
+                    Constraint,
+                    PreTransformationConstraint,
+                    ModelWrapper,
+                ),
+            ):
+                for key in obj.__dict__:
+                    s_obj = obj.__dict__[key]
+                    if id(s_obj) not in visited:
+                        to_cuda(s_obj)
+            elif isinstance(obj, (list, tuple)):
+                for item in obj:
+                    if id(item) not in visited and isinstance(
+                        item, (Transformation, Constraint, PreTransformationConstraint)
+                    ):
+                        to_cuda(item)
+
+        to_cuda(self)
+
+    def get_indices_to_order(self, current_text, **kwargs):
+        """Applies ``pre_transformation_constraints`` to ``text`` to get all
+        the indices that can be used to search and order.
+
+        Args:
+            current_text: The current ``AttackedText`` for which we need to find indices are eligible to be ordered.
+        Returns:
+            The length and the filtered list of indices which search methods can use to search/order.
+        """
+
+        indices_to_order = self.transformation(
+            current_text,
+            pre_transformation_constraints=self.pre_transformation_constraints,
+            return_indices=True,
+            **kwargs,
+        )
+
+        len_text = len(indices_to_order)
+
+        # Convert indices_to_order to list for easier shuffling later
+        return len_text, list(indices_to_order)
+
+    def _get_transformations_uncached(self, current_text, original_text=None, **kwargs):
+        """Applies ``self.transformation`` to ``text``, then filters the list
+        of possible transformations through the applicable constraints.
+
+        Args:
+            current_text: The current ``AttackedText`` on which to perform the transformations.
+            original_text: The original ``AttackedText`` from which the attack started.
+        Returns:
+            A filtered list of transformations where each transformation matches the constraints
+        """
+        transformed_texts = self.transformation(
+            current_text,
+            pre_transformation_constraints=self.pre_transformation_constraints,
+            **kwargs,
+        )
+
+        return transformed_texts
+
+    def get_transformations(self, current_text, original_text=None, **kwargs):
+        """Applies ``self.transformation`` to ``text``, then filters the list
+        of possible transformations through the applicable constraints.
+
+        Args:
+            current_text: The current ``AttackedText`` on which to perform the transformations.
+            original_text: The original ``AttackedText`` from which the attack started.
+        Returns:
+            A filtered list of transformations where each transformation matches the constraints
+        """
+        if not self.transformation:
+            raise RuntimeError(
+                "Cannot call `get_transformations` without a transformation."
+            )
+
+        if self.use_transformation_cache:
+            cache_key = tuple([current_text] + sorted(kwargs.items()))
+            if utils.hashable(cache_key) and cache_key in self.transformation_cache:
+                # promote transformed_text to the top of the LRU cache
+                self.transformation_cache[cache_key] = self.transformation_cache[
+                    cache_key
+                ]
+                transformed_texts = list(self.transformation_cache[cache_key])
+            else:
+                transformed_texts = self._get_transformations_uncached(
+                    current_text, original_text, **kwargs
+                )
+                if utils.hashable(cache_key):
+                    self.transformation_cache[cache_key] = tuple(transformed_texts)
+        else:
+            transformed_texts = self._get_transformations_uncached(
+                current_text, original_text, **kwargs
+            )
+
+        return self.filter_transformations(
+            transformed_texts, current_text, original_text
+        )
+
+    def _filter_transformations_uncached(
+        self, transformed_texts, current_text, original_text=None
+    ):
+        """Filters a list of potential transformed texts based on
+        ``self.constraints``
+
+        Args:
+            transformed_texts: A list of candidate transformed ``AttackedText`` to filter.
+            current_text: The current ``AttackedText`` on which the transformation was applied.
+            original_text: The original ``AttackedText`` from which the attack started.
+        """
+        filtered_texts = transformed_texts[:]
+        for C in self.constraints:
+            if len(filtered_texts) == 0:
+                break
+            if C.compare_against_original:
+                if not original_text:
+                    raise ValueError(
+                        f"Missing `original_text` argument when constraint {type(C)} is set to compare against `original_text`"
+                    )
+
+                filtered_texts = C.call_many(filtered_texts, original_text)
+            else:
+                filtered_texts = C.call_many(filtered_texts, current_text)
+        # Default to false for all original transformations.
+        for original_transformed_text in transformed_texts:
+            self.constraints_cache[(current_text, original_transformed_text)] = False
+        # Set unfiltered transformations to True in the cache.
+        for filtered_text in filtered_texts:
+            self.constraints_cache[(current_text, filtered_text)] = True
+        return filtered_texts
+
+    def filter_transformations(
+        self, transformed_texts, current_text, original_text=None
+    ):
+        """Filters a list of potential transformed texts based on
+        ``self.constraints`` Utilizes an LRU cache to attempt to avoid
+        recomputing common transformations.
+
+        Args:
+            transformed_texts: A list of candidate transformed ``AttackedText`` to filter.
+            current_text: The current ``AttackedText`` on which the transformation was applied.
+            original_text: The original ``AttackedText`` from which the attack started.
+        """
+        # Remove any occurences of current_text in transformed_texts
+        transformed_texts = [
+            t for t in transformed_texts if t.text != current_text.text
+        ]
+        # Populate cache with transformed_texts
+        uncached_texts = []
+        filtered_texts = []
+        for transformed_text in transformed_texts:
+            if (current_text, transformed_text) not in self.constraints_cache:
+                uncached_texts.append(transformed_text)
+            else:
+                # promote transformed_text to the top of the LRU cache
+                self.constraints_cache[
+                    (current_text, transformed_text)
+                ] = self.constraints_cache[(current_text, transformed_text)]
+                if self.constraints_cache[(current_text, transformed_text)]:
+                    filtered_texts.append(transformed_text)
+        filtered_texts += self._filter_transformations_uncached(
+            uncached_texts, current_text, original_text=original_text
+        )
+        # Sort transformations to ensure order is preserved between runs
+        filtered_texts.sort(key=lambda t: t.text)
+        return filtered_texts
+
+    def _attack(self, initial_result):
+        """Calls the ``SearchMethod`` to perturb the ``AttackedText`` stored in
+        ``initial_result``.
+
+        Args:
+            initial_result: The initial ``GoalFunctionResult`` from which to perturb.
+
+        Returns:
+            A ``SuccessfulAttackResult``, ``FailedAttackResult``,
+                or ``MaximizedAttackResult``.
+        """
+        final_result = self.search_method(initial_result)
+        self.clear_cache()
+        if final_result.goal_status == GoalFunctionResultStatus.SUCCEEDED:
+            result = SuccessfulAttackResult(
+                initial_result,
+                final_result,
+            )
+        elif final_result.goal_status == GoalFunctionResultStatus.SEARCHING:
+            result = FailedAttackResult(
+                initial_result,
+                final_result,
+            )
+        elif final_result.goal_status == GoalFunctionResultStatus.MAXIMIZING:
+            result = MaximizedAttackResult(
+                initial_result,
+                final_result,
+            )
+        else:
+            raise ValueError(f"Unrecognized goal status {final_result.goal_status}")
+        return result
+
+    def attack(self, example, ground_truth_output):
+        """Attack a single example.
+
+        Args:
+            example (:obj:`str`, :obj:`OrderedDict[str, str]` or :class:`~textattack.shared.AttackedText`):
+                Example to attack. It can be a single string or an `OrderedDict` where
+                keys represent the input fields (e.g. "premise", "hypothesis") and the values are the actual input textx.
+                Also accepts :class:`~textattack.shared.AttackedText` that wraps around the input.
+            ground_truth_output(:obj:`int`, :obj:`float` or :obj:`str`):
+                Ground truth output of `example`.
+                For classification tasks, it should be an integer representing the ground truth label.
+                For regression tasks (e.g. STS), it should be the target value.
+                For seq2seq tasks (e.g. translation), it should be the target string.
+        Returns:
+            :class:`~textattack.attack_results.AttackResult` that represents the result of the attack.
+        """
+        assert isinstance(
+            example, (str, OrderedDict, AttackedText)
+        ), "`example` must either be `str`, `collections.OrderedDict`, `textattack.shared.AttackedText`."
+        if isinstance(example, (str, OrderedDict)):
+            example = AttackedText(example)
+
+        assert isinstance(
+            ground_truth_output, (int, str)
+        ), "`ground_truth_output` must either be `str` or `int`."
+        goal_function_result, _ = self.goal_function.init_attack_example(
+            example, ground_truth_output
+        )
+        if goal_function_result.goal_status == GoalFunctionResultStatus.SKIPPED:
+            return SkippedAttackResult(goal_function_result)
+        else:
+            result = self._attack(goal_function_result)
+            return result
+
+    def __repr__(self):
+        """Prints attack parameters in a human-readable string.
+
+        Inspired by the readability of printing PyTorch nn.Modules:
+        https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/module.py
+        """
+        main_str = "Attack" + "("
+        lines = []
+
+        lines.append(utils.add_indent(f"(search_method): {self.search_method}", 2))
+        # self.goal_function
+        lines.append(utils.add_indent(f"(goal_function):  {self.goal_function}", 2))
+        # self.transformation
+        lines.append(utils.add_indent(f"(transformation):  {self.transformation}", 2))
+        # self.constraints
+        constraints_lines = []
+        constraints = self.constraints + self.pre_transformation_constraints
+        if len(constraints):
+            for i, constraint in enumerate(constraints):
+                constraints_lines.append(utils.add_indent(f"({i}): {constraint}", 2))
+            constraints_str = utils.add_indent("\n" + "\n".join(constraints_lines), 2)
+        else:
+            constraints_str = "None"
+        lines.append(utils.add_indent(f"(constraints): {constraints_str}", 2))
+        # self.is_black_box
+        lines.append(utils.add_indent(f"(is_black_box):  {self.is_black_box}", 2))
+        main_str += "\n  " + "\n  ".join(lines) + "\n"
+        main_str += ")"
+        return main_str
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["transformation_cache"] = None
+        state["constraints_cache"] = None
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+        self.transformation_cache = lru.LRU(self.transformation_cache_size)
+        self.constraints_cache = lru.LRU(self.constraint_cache_size)
+
+    __str__ = __repr__

textattack/attack_args.py

@@ -0,0 +1,763 @@
+"""
+AttackArgs Class
+================
+"""
+
+from dataclasses import dataclass, field
+import json
+import os
+import sys
+import time
+from typing import Dict, Optional
+
+import textattack
+from textattack.shared.utils import ARGS_SPLIT_TOKEN, load_module_from_file
+
+from .attack import Attack
+from .dataset_args import DatasetArgs
+from .model_args import ModelArgs
+
+ATTACK_RECIPE_NAMES = {
+    "alzantot": "textattack.attack_recipes.GeneticAlgorithmAlzantot2018",
+    "bae": "textattack.attack_recipes.BAEGarg2019",
+    "bert-attack": "textattack.attack_recipes.BERTAttackLi2020",
+    "faster-alzantot": "textattack.attack_recipes.FasterGeneticAlgorithmJia2019",
+    "deepwordbug": "textattack.attack_recipes.DeepWordBugGao2018",
+    "hotflip": "textattack.attack_recipes.HotFlipEbrahimi2017",
+    "input-reduction": "textattack.attack_recipes.InputReductionFeng2018",
+    "kuleshov": "textattack.attack_recipes.Kuleshov2017",
+    "morpheus": "textattack.attack_recipes.MorpheusTan2020",
+    "seq2sick": "textattack.attack_recipes.Seq2SickCheng2018BlackBox",
+    "textbugger": "textattack.attack_recipes.TextBuggerLi2018",
+    "textfooler": "textattack.attack_recipes.TextFoolerJin2019",
+    "pwws": "textattack.attack_recipes.PWWSRen2019",
+    "iga": "textattack.attack_recipes.IGAWang2019",
+    "pruthi": "textattack.attack_recipes.Pruthi2019",
+    "pso": "textattack.attack_recipes.PSOZang2020",
+    "checklist": "textattack.attack_recipes.CheckList2020",
+    "clare": "textattack.attack_recipes.CLARE2020",
+    "a2t": "textattack.attack_recipes.A2TYoo2021",
+}
+
+
+BLACK_BOX_TRANSFORMATION_CLASS_NAMES = {
+    "random-synonym-insertion": "textattack.transformations.RandomSynonymInsertion",
+    "word-deletion": "textattack.transformations.WordDeletion",
+    "word-swap-embedding": "textattack.transformations.WordSwapEmbedding",
+    "word-swap-homoglyph": "textattack.transformations.WordSwapHomoglyphSwap",
+    "word-swap-inflections": "textattack.transformations.WordSwapInflections",
+    "word-swap-neighboring-char-swap": "textattack.transformations.WordSwapNeighboringCharacterSwap",
+    "word-swap-random-char-deletion": "textattack.transformations.WordSwapRandomCharacterDeletion",
+    "word-swap-random-char-insertion": "textattack.transformations.WordSwapRandomCharacterInsertion",
+    "word-swap-random-char-substitution": "textattack.transformations.WordSwapRandomCharacterSubstitution",
+    "word-swap-wordnet": "textattack.transformations.WordSwapWordNet",
+    "word-swap-masked-lm": "textattack.transformations.WordSwapMaskedLM",
+    "word-swap-hownet": "textattack.transformations.WordSwapHowNet",
+    "word-swap-qwerty": "textattack.transformations.WordSwapQWERTY",
+}
+
+
+WHITE_BOX_TRANSFORMATION_CLASS_NAMES = {
+    "word-swap-gradient": "textattack.transformations.WordSwapGradientBased"
+}
+
+
+CONSTRAINT_CLASS_NAMES = {
+    #
+    # Semantics constraints
+    #
+    "embedding": "textattack.constraints.semantics.WordEmbeddingDistance",
+    "bert": "textattack.constraints.semantics.sentence_encoders.BERT",
+    "infer-sent": "textattack.constraints.semantics.sentence_encoders.InferSent",
+    "thought-vector": "textattack.constraints.semantics.sentence_encoders.ThoughtVector",
+    "use": "textattack.constraints.semantics.sentence_encoders.UniversalSentenceEncoder",
+    "muse": "textattack.constraints.semantics.sentence_encoders.MultilingualUniversalSentenceEncoder",
+    "bert-score": "textattack.constraints.semantics.BERTScore",
+    #
+    # Grammaticality constraints
+    #
+    "lang-tool": "textattack.constraints.grammaticality.LanguageTool",
+    "part-of-speech": "textattack.constraints.grammaticality.PartOfSpeech",
+    "goog-lm": "textattack.constraints.grammaticality.language_models.GoogleLanguageModel",
+    "gpt2": "textattack.constraints.grammaticality.language_models.GPT2",
+    "learning-to-write": "textattack.constraints.grammaticality.language_models.LearningToWriteLanguageModel",
+    "cola": "textattack.constraints.grammaticality.COLA",
+    #
+    # Overlap constraints
+    #
+    "bleu": "textattack.constraints.overlap.BLEU",
+    "chrf": "textattack.constraints.overlap.chrF",
+    "edit-distance": "textattack.constraints.overlap.LevenshteinEditDistance",
+    "meteor": "textattack.constraints.overlap.METEOR",
+    "max-words-perturbed": "textattack.constraints.overlap.MaxWordsPerturbed",
+    #
+    # Pre-transformation constraints
+    #
+    "repeat": "textattack.constraints.pre_transformation.RepeatModification",
+    "stopword": "textattack.constraints.pre_transformation.StopwordModification",
+    "max-word-index": "textattack.constraints.pre_transformation.MaxWordIndexModification",
+}
+
+
+SEARCH_METHOD_CLASS_NAMES = {
+    "beam-search": "textattack.search_methods.BeamSearch",
+    "greedy": "textattack.search_methods.GreedySearch",
+    "ga-word": "textattack.search_methods.GeneticAlgorithm",
+    "greedy-word-wir": "textattack.search_methods.GreedyWordSwapWIR",
+    "pso": "textattack.search_methods.ParticleSwarmOptimization",
+}
+
+
+GOAL_FUNCTION_CLASS_NAMES = {
+    #
+    # Classification goal functions
+    #
+    "targeted-classification": "textattack.goal_functions.classification.TargetedClassification",
+    "untargeted-classification": "textattack.goal_functions.classification.UntargetedClassification",
+    "input-reduction": "textattack.goal_functions.classification.InputReduction",
+    #
+    # Text goal functions
+    #
+    "minimize-bleu": "textattack.goal_functions.text.MinimizeBleu",
+    "non-overlapping-output": "textattack.goal_functions.text.NonOverlappingOutput",
+    "text-to-text": "textattack.goal_functions.text.TextToTextGoalFunction",
+}
+
+
+@dataclass
+class AttackArgs:
+    """Attack arguments to be passed to :class:`~textattack.Attacker`.
+
+    Args:
+        num_examples (:obj:`int`, 'optional`, defaults to :obj:`10`):
+            The number of examples to attack. :obj:`-1` for entire dataset.
+        num_successful_examples (:obj:`int`, `optional`, defaults to :obj:`None`):
+            The number of successful adversarial examples we want. This is different from :obj:`num_examples`
+            as :obj:`num_examples` only cares about attacking `N` samples while :obj:`num_successful_examples` aims to keep attacking
+            until we have `N` successful cases.
+
+            .. note::
+                If set, this argument overrides `num_examples` argument.
+        num_examples_offset (:obj: `int`, `optional`, defaults to :obj:`0`):
+            The offset index to start at in the dataset.
+        attack_n (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether to run attack until total of `N` examples have been attacked (and not skipped).
+        shuffle (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If :obj:`True`, we randomly shuffle the dataset before attacking. However, this avoids actually shuffling
+            the dataset internally and opts for shuffling the list of indices of examples we want to attack. This means
+            :obj:`shuffle` can now be used with checkpoint saving.
+        query_budget (:obj:`int`, `optional`, defaults to :obj:`None`):
+            The maximum number of model queries allowed per example attacked.
+            If not set, we use the query budget set in the :class:`~textattack.goal_functions.GoalFunction` object (which by default is :obj:`float("inf")`).
+
+            .. note::
+                Setting this overwrites the query budget set in :class:`~textattack.goal_functions.GoalFunction` object.
+        checkpoint_interval (:obj:`int`, `optional`, defaults to :obj:`None`):
+            If set, checkpoint will be saved after attacking every `N` examples. If :obj:`None` is passed, no checkpoints will be saved.
+        checkpoint_dir (:obj:`str`, `optional`, defaults to :obj:`"checkpoints"`):
+            The directory to save checkpoint files.
+        random_seed (:obj:`int`, `optional`, defaults to :obj:`765`):
+            Random seed for reproducibility.
+        parallel (:obj:`False`, `optional`, defaults to :obj:`False`):
+            If :obj:`True`, run attack using multiple CPUs/GPUs.
+        num_workers_per_device (:obj:`int`, `optional`, defaults to :obj:`1`):
+            Number of worker processes to run per device in parallel mode (i.e. :obj:`parallel=True`). For example, if you are using GPUs and :obj:`num_workers_per_device=2`,
+            then 2 processes will be running in each GPU.
+        log_to_txt (:obj:`str`, `optional`, defaults to :obj:`None`):
+            If set, save attack logs as a `.txt` file to the directory specified by this argument.
+            If the last part of the provided path ends with `.txt` extension, it is assumed to the desired path of the log file.
+        log_to_csv (:obj:`str`, `optional`, defaults to :obj:`None`):
+            If set, save attack logs as a CSV file to the directory specified by this argument.
+            If the last part of the provided path ends with `.csv` extension, it is assumed to the desired path of the log file.
+        csv_coloring_style (:obj:`str`, `optional`, defaults to :obj:`"file"`):
+            Method for choosing how to mark perturbed parts of the text. Options are :obj:`"file"`, :obj:`"plain"`, and :obj:`"html"`.
+            :obj:`"file"` wraps perturbed parts with double brackets :obj:`[[ <text> ]]` while :obj:`"plain"` does not mark the text in any way.
+        log_to_visdom (:obj:`dict`, `optional`, defaults to :obj:`None`):
+            If set, Visdom logger is used with the provided dictionary passed as a keyword arguments to :class:`~textattack.loggers.VisdomLogger`.
+            Pass in empty dictionary to use default arguments. For custom logger, the dictionary should have the following
+            three keys and their corresponding values: :obj:`"env", "port", "hostname"`.
+        log_to_wandb(:obj:`dict`, `optional`, defaults to :obj:`None`):
+            If set, WandB logger is used with the provided dictionary passed as a keyword arguments to :class:`~textattack.loggers.WeightsAndBiasesLogger`.
+            Pass in empty dictionary to use default arguments. For custom logger, the dictionary should have the following
+            key and its corresponding value: :obj:`"project"`.
+        disable_stdout (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Disable displaying individual attack results to stdout.
+        silent (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Disable all logging (except for errors). This is stronger than :obj:`disable_stdout`.
+        enable_advance_metrics (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Enable calculation and display of optional advance post-hoc metrics like perplexity, grammar errors, etc.
+    """
+
+    num_examples: int = 10
+    num_successful_examples: int = None
+    num_examples_offset: int = 0
+    attack_n: bool = False
+    shuffle: bool = False
+    query_budget: int = None
+    checkpoint_interval: int = None
+    checkpoint_dir: str = "checkpoints"
+    random_seed: int = 765  # equivalent to sum((ord(c) for c in "TEXTATTACK"))
+    parallel: bool = False
+    num_workers_per_device: int = 1
+    log_to_txt: str = None
+    log_to_csv: str = None
+    log_summary_to_json: str = None
+    csv_coloring_style: str = "file"
+    log_to_visdom: dict = None
+    log_to_wandb: dict = None
+    disable_stdout: bool = False
+    silent: bool = False
+    enable_advance_metrics: bool = False
+    metrics: Optional[Dict] = None
+
+    def __post_init__(self):
+        if self.num_successful_examples:
+            self.num_examples = None
+        if self.num_examples:
+            assert (
+                self.num_examples >= 0 or self.num_examples == -1
+            ), "`num_examples` must be greater than or equal to 0 or equal to -1."
+        if self.num_successful_examples:
+            assert (
+                self.num_successful_examples >= 0
+            ), "`num_examples` must be greater than or equal to 0."
+
+        if self.query_budget:
+            assert self.query_budget > 0, "`query_budget` must be greater than 0."
+
+        if self.checkpoint_interval:
+            assert (
+                self.checkpoint_interval > 0
+            ), "`checkpoint_interval` must be greater than 0."
+
+        assert (
+            self.num_workers_per_device > 0
+        ), "`num_workers_per_device` must be greater than 0."
+
+    @classmethod
+    def _add_parser_args(cls, parser):
+        """Add listed args to command line parser."""
+        default_obj = cls()
+        num_ex_group = parser.add_mutually_exclusive_group(required=False)
+        num_ex_group.add_argument(
+            "--num-examples",
+            "-n",
+            type=int,
+            default=default_obj.num_examples,
+            help="The number of examples to process, -1 for entire dataset.",
+        )
+        num_ex_group.add_argument(
+            "--num-successful-examples",
+            type=int,
+            default=default_obj.num_successful_examples,
+            help="The number of successful adversarial examples we want.",
+        )
+        parser.add_argument(
+            "--num-examples-offset",
+            "-o",
+            type=int,
+            required=False,
+            default=default_obj.num_examples_offset,
+            help="The offset to start at in the dataset.",
+        )
+        parser.add_argument(
+            "--query-budget",
+            "-q",
+            type=int,
+            default=default_obj.query_budget,
+            help="The maximum number of model queries allowed per example attacked. Setting this overwrites the query budget set in `GoalFunction` object.",
+        )
+        parser.add_argument(
+            "--shuffle",
+            action="store_true",
+            default=default_obj.shuffle,
+            help="If `True`, shuffle the samples before we attack the dataset. Default is False.",
+        )
+        parser.add_argument(
+            "--attack-n",
+            action="store_true",
+            default=default_obj.attack_n,
+            help="Whether to run attack until `n` examples have been attacked (not skipped).",
+        )
+        parser.add_argument(
+            "--checkpoint-dir",
+            required=False,
+            type=str,
+            default=default_obj.checkpoint_dir,
+            help="The directory to save checkpoint files.",
+        )
+        parser.add_argument(
+            "--checkpoint-interval",
+            required=False,
+            type=int,
+            default=default_obj.checkpoint_interval,
+            help="If set, checkpoint will be saved after attacking every N examples. If not set, no checkpoints will be saved.",
+        )
+        parser.add_argument(
+            "--random-seed",
+            default=default_obj.random_seed,
+            type=int,
+            help="Random seed for reproducibility.",
+        )
+        parser.add_argument(
+            "--parallel",
+            action="store_true",
+            default=default_obj.parallel,
+            help="Run attack using multiple GPUs.",
+        )
+        parser.add_argument(
+            "--num-workers-per-device",
+            default=default_obj.num_workers_per_device,
+            type=int,
+            help="Number of worker processes to run per device.",
+        )
+        parser.add_argument(
+            "--log-to-txt",
+            nargs="?",
+            default=default_obj.log_to_txt,
+            const="",
+            type=str,
+            help="Path to which to save attack logs as a text file. Set this argument if you want to save text logs. "
+            "If the last part of the path ends with `.txt` extension, the path is assumed to path for output file.",
+        )
+        parser.add_argument(
+            "--log-to-csv",
+            nargs="?",
+            default=default_obj.log_to_csv,
+            const="",
+            type=str,
+            help="Path to which to save attack logs as a CSV file. Set this argument if you want to save CSV logs. "
+            "If the last part of the path ends with `.csv` extension, the path is assumed to path for output file.",
+        )
+        parser.add_argument(
+            "--log-summary-to-json",
+            nargs="?",
+            default=default_obj.log_summary_to_json,
+            const="",
+            type=str,
+            help="Path to which to save attack summary as a JSON file. Set this argument if you want to save attack results summary in a JSON. "
+            "If the last part of the path ends with `.json` extension, the path is assumed to path for output file.",
+        )
+        parser.add_argument(
+            "--csv-coloring-style",
+            default=default_obj.csv_coloring_style,
+            type=str,
+            help='Method for choosing how to mark perturbed parts of the text in CSV logs. Options are "file" and "plain". '
+            '"file" wraps text with double brackets `[[ <text> ]]` while "plain" does not mark any text. Default is "file".',
+        )
+        parser.add_argument(
+            "--log-to-visdom",
+            nargs="?",
+            default=None,
+            const='{"env": "main", "port": 8097, "hostname": "localhost"}',
+            type=json.loads,
+            help="Set this argument if you want to log attacks to Visdom. The dictionary should have the following "
+            'three keys and their corresponding values: `"env", "port", "hostname"`. '
+            'Example for command line use: `--log-to-visdom {"env": "main", "port": 8097, "hostname": "localhost"}`.',
+        )
+        parser.add_argument(
+            "--log-to-wandb",
+            nargs="?",
+            default=None,
+            const='{"project": "textattack"}',
+            type=json.loads,
+            help="Set this argument if you want to log attacks to WandB. The dictionary should have the following "
+            'key and its corresponding value: `"project"`. '
+            'Example for command line use: `--log-to-wandb {"project": "textattack"}`.',
+        )
+        parser.add_argument(
+            "--disable-stdout",
+            action="store_true",
+            default=default_obj.disable_stdout,
+            help="Disable logging attack results to stdout",
+        )
+        parser.add_argument(
+            "--silent",
+            action="store_true",
+            default=default_obj.silent,
+            help="Disable all logging",
+        )
+        parser.add_argument(
+            "--enable-advance-metrics",
+            action="store_true",
+            default=default_obj.enable_advance_metrics,
+            help="Enable calculation and display of optional advance post-hoc metrics like perplexity, USE distance, etc.",
+        )
+
+        return parser
+
+    @classmethod
+    def create_loggers_from_args(cls, args):
+        """Creates AttackLogManager from an AttackArgs object."""
+        assert isinstance(
+            args, cls
+        ), f"Expect args to be of type `{type(cls)}`, but got type `{type(args)}`."
+
+        # Create logger
+        attack_log_manager = textattack.loggers.AttackLogManager(args.metrics)
+
+        # Get current time for file naming
+        timestamp = time.strftime("%Y-%m-%d-%H-%M")
+
+        # if '--log-to-txt' specified with arguments
+        if args.log_to_txt is not None:
+            if args.log_to_txt.lower().endswith(".txt"):
+                txt_file_path = args.log_to_txt
+            else:
+                txt_file_path = os.path.join(args.log_to_txt, f"{timestamp}-log.txt")
+
+            dir_path = os.path.dirname(txt_file_path)
+            dir_path = dir_path if dir_path else "."
+            if not os.path.exists(dir_path):
+                os.makedirs(os.path.dirname(txt_file_path))
+
+            color_method = "file"
+            attack_log_manager.add_output_file(txt_file_path, color_method)
+
+        # if '--log-to-csv' specified with arguments
+        if args.log_to_csv is not None:
+            if args.log_to_csv.lower().endswith(".csv"):
+                csv_file_path = args.log_to_csv
+            else:
+                csv_file_path = os.path.join(args.log_to_csv, f"{timestamp}-log.csv")
+
+            dir_path = os.path.dirname(csv_file_path)
+            dir_path = dir_path if dir_path else "."
+            if not os.path.exists(dir_path):
+                os.makedirs(dir_path)
+
+            color_method = (
+                None if args.csv_coloring_style == "plain" else args.csv_coloring_style
+            )
+            attack_log_manager.add_output_csv(csv_file_path, color_method)
+
+        # if '--log-summary-to-json' specified with arguments
+        if args.log_summary_to_json is not None:
+            if args.log_summary_to_json.lower().endswith(".json"):
+                summary_json_file_path = args.log_summary_to_json
+            else:
+                summary_json_file_path = os.path.join(
+                    args.log_summary_to_json, f"{timestamp}-attack_summary_log.json"
+                )
+
+            dir_path = os.path.dirname(summary_json_file_path)
+            dir_path = dir_path if dir_path else "."
+            if not os.path.exists(dir_path):
+                os.makedirs(os.path.dirname(summary_json_file_path))
+
+            attack_log_manager.add_output_summary_json(summary_json_file_path)
+
+        # Visdom
+        if args.log_to_visdom is not None:
+            attack_log_manager.enable_visdom(**args.log_to_visdom)
+
+        # Weights & Biases
+        if args.log_to_wandb is not None:
+            attack_log_manager.enable_wandb(**args.log_to_wandb)
+
+        # Stdout
+        if not args.disable_stdout and not sys.stdout.isatty():
+            attack_log_manager.disable_color()
+        elif not args.disable_stdout:
+            attack_log_manager.enable_stdout()
+
+        return attack_log_manager
+
+
+@dataclass
+class _CommandLineAttackArgs:
+    """Attack args for command line execution. This requires more arguments to
+    create ``Attack`` object as specified.
+
+    Args:
+        transformation (:obj:`str`, `optional`, defaults to :obj:`"word-swap-embedding"`):
+            Name of transformation to use.
+        constraints (:obj:`list[str]`, `optional`, defaults to :obj:`["repeat", "stopword"]`):
+            List of names of constraints to use.
+        goal_function (:obj:`str`, `optional`, defaults to :obj:`"untargeted-classification"`):
+            Name of goal function to use.
+        search_method (:obj:`str`, `optional`, defualts to :obj:`"greedy-word-wir"`):
+            Name of search method to use.
+        attack_recipe (:obj:`str`, `optional`, defaults to :obj:`None`):
+            Name of attack recipe to use.
+            .. note::
+                Setting this overrides any previous selection of transformation, constraints, goal function, and search method.
+        attack_from_file (:obj:`str`, `optional`, defaults to :obj:`None`):
+            Path of `.py` file from which to load attack from. Use `<path>^<variable_name>` to specifiy which variable to import from the file.
+            .. note::
+                If this is set, it overrides any previous selection of transformation, constraints, goal function, and search method
+        interactive (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If `True`, carry attack in interactive mode.
+        parallel (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If `True`, attack in parallel.
+        model_batch_size (:obj:`int`, `optional`, defaults to :obj:`32`):
+            The batch size for making queries to the victim model.
+        model_cache_size (:obj:`int`, `optional`, defaults to :obj:`2**18`):
+            The maximum number of items to keep in the model results cache at once.
+        constraint-cache-size (:obj:`int`, `optional`, defaults to :obj:`2**18`):
+            The maximum number of items to keep in the constraints cache at once.
+    """
+
+    transformation: str = "word-swap-embedding"
+    constraints: list = field(default_factory=lambda: ["repeat", "stopword"])
+    goal_function: str = "untargeted-classification"
+    search_method: str = "greedy-word-wir"
+    attack_recipe: str = None
+    attack_from_file: str = None
+    interactive: bool = False
+    parallel: bool = False
+    model_batch_size: int = 32
+    model_cache_size: int = 2**18
+    constraint_cache_size: int = 2**18
+
+    @classmethod
+    def _add_parser_args(cls, parser):
+        """Add listed args to command line parser."""
+        default_obj = cls()
+        transformation_names = set(BLACK_BOX_TRANSFORMATION_CLASS_NAMES.keys()) | set(
+            WHITE_BOX_TRANSFORMATION_CLASS_NAMES.keys()
+        )
+        parser.add_argument(
+            "--transformation",
+            type=str,
+            required=False,
+            default=default_obj.transformation,
+            help='The transformation to apply. Usage: "--transformation {transformation}:{arg_1}={value_1},{arg_3}={value_3}". Choices: '
+            + str(transformation_names),
+        )
+        parser.add_argument(
+            "--constraints",
+            type=str,
+            required=False,
+            nargs="*",
+            default=default_obj.constraints,
+            help='Constraints to add to the attack. Usage: "--constraints {constraint}:{arg_1}={value_1},{arg_3}={value_3}". Choices: '
+            + str(CONSTRAINT_CLASS_NAMES.keys()),
+        )
+        goal_function_choices = ", ".join(GOAL_FUNCTION_CLASS_NAMES.keys())
+        parser.add_argument(
+            "--goal-function",
+            "-g",
+            default=default_obj.goal_function,
+            help=f"The goal function to use. choices: {goal_function_choices}",
+        )
+        attack_group = parser.add_mutually_exclusive_group(required=False)
+        search_choices = ", ".join(SEARCH_METHOD_CLASS_NAMES.keys())
+        attack_group.add_argument(
+            "--search-method",
+            "--search",
+            "-s",
+            type=str,
+            required=False,
+            default=default_obj.search_method,
+            help=f"The search method to use. choices: {search_choices}",
+        )
+        attack_group.add_argument(
+            "--attack-recipe",
+            "--recipe",
+            "-r",
+            type=str,
+            required=False,
+            default=default_obj.attack_recipe,
+            help="full attack recipe (overrides provided goal function, transformation & constraints)",
+            choices=ATTACK_RECIPE_NAMES.keys(),
+        )
+        attack_group.add_argument(
+            "--attack-from-file",
+            type=str,
+            required=False,
+            default=default_obj.attack_from_file,
+            help="Path of `.py` file from which to load attack from. Use `<path>^<variable_name>` to specifiy which variable to import from the file.",
+        )
+        parser.add_argument(
+            "--interactive",
+            action="store_true",
+            default=default_obj.interactive,
+            help="Whether to run attacks interactively.",
+        )
+        parser.add_argument(
+            "--model-batch-size",
+            type=int,
+            default=default_obj.model_batch_size,
+            help="The batch size for making calls to the model.",
+        )
+        parser.add_argument(
+            "--model-cache-size",
+            type=int,
+            default=default_obj.model_cache_size,
+            help="The maximum number of items to keep in the model results cache at once.",
+        )
+        parser.add_argument(
+            "--constraint-cache-size",
+            type=int,
+            default=default_obj.constraint_cache_size,
+            help="The maximum number of items to keep in the constraints cache at once.",
+        )
+
+        return parser
+
+    @classmethod
+    def _create_transformation_from_args(cls, args, model_wrapper):
+        """Create `Transformation` based on provided `args` and
+        `model_wrapper`."""
+
+        transformation_name = args.transformation
+        if ARGS_SPLIT_TOKEN in transformation_name:
+            transformation_name, params = transformation_name.split(ARGS_SPLIT_TOKEN)
+
+            if transformation_name in WHITE_BOX_TRANSFORMATION_CLASS_NAMES:
+                transformation = eval(
+                    f"{WHITE_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}(model_wrapper.model, {params})"
+                )
+            elif transformation_name in BLACK_BOX_TRANSFORMATION_CLASS_NAMES:
+                transformation = eval(
+                    f"{BLACK_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}({params})"
+                )
+            else:
+                raise ValueError(
+                    f"Error: unsupported transformation {transformation_name}"
+                )
+        else:
+            if transformation_name in WHITE_BOX_TRANSFORMATION_CLASS_NAMES:
+                transformation = eval(
+                    f"{WHITE_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}(model_wrapper.model)"
+                )
+            elif transformation_name in BLACK_BOX_TRANSFORMATION_CLASS_NAMES:
+                transformation = eval(
+                    f"{BLACK_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}()"
+                )
+            else:
+                raise ValueError(
+                    f"Error: unsupported transformation {transformation_name}"
+                )
+        return transformation
+
+    @classmethod
+    def _create_goal_function_from_args(cls, args, model_wrapper):
+        """Create `GoalFunction` based on provided `args` and
+        `model_wrapper`."""
+
+        goal_function = args.goal_function
+        if ARGS_SPLIT_TOKEN in goal_function:
+            goal_function_name, params = goal_function.split(ARGS_SPLIT_TOKEN)
+            if goal_function_name not in GOAL_FUNCTION_CLASS_NAMES:
+                raise ValueError(
+                    f"Error: unsupported goal_function {goal_function_name}"
+                )
+            goal_function = eval(
+                f"{GOAL_FUNCTION_CLASS_NAMES[goal_function_name]}(model_wrapper, {params})"
+            )
+        elif goal_function in GOAL_FUNCTION_CLASS_NAMES:
+            goal_function = eval(
+                f"{GOAL_FUNCTION_CLASS_NAMES[goal_function]}(model_wrapper)"
+            )
+        else:
+            raise ValueError(f"Error: unsupported goal_function {goal_function}")
+        if args.query_budget:
+            goal_function.query_budget = args.query_budget
+        goal_function.model_cache_size = args.model_cache_size
+        goal_function.batch_size = args.model_batch_size
+        return goal_function
+
+    @classmethod
+    def _create_constraints_from_args(cls, args):
+        """Create list of `Constraints` based on provided `args`."""
+
+        if not args.constraints:
+            return []
+
+        _constraints = []
+        for constraint in args.constraints:
+            if ARGS_SPLIT_TOKEN in constraint:
+                constraint_name, params = constraint.split(ARGS_SPLIT_TOKEN)
+                if constraint_name not in CONSTRAINT_CLASS_NAMES:
+                    raise ValueError(f"Error: unsupported constraint {constraint_name}")
+                _constraints.append(
+                    eval(f"{CONSTRAINT_CLASS_NAMES[constraint_name]}({params})")
+                )
+            elif constraint in CONSTRAINT_CLASS_NAMES:
+                _constraints.append(eval(f"{CONSTRAINT_CLASS_NAMES[constraint]}()"))
+            else:
+                raise ValueError(f"Error: unsupported constraint {constraint}")
+
+        return _constraints
+
+    @classmethod
+    def _create_attack_from_args(cls, args, model_wrapper):
+        """Given ``CommandLineArgs`` and ``ModelWrapper``, return specified
+        ``Attack`` object."""
+
+        assert isinstance(
+            args, cls
+        ), f"Expect args to be of type `{type(cls)}`, but got type `{type(args)}`."
+
+        if args.attack_recipe:
+            if ARGS_SPLIT_TOKEN in args.attack_recipe:
+                recipe_name, params = args.attack_recipe.split(ARGS_SPLIT_TOKEN)
+                if recipe_name not in ATTACK_RECIPE_NAMES:
+                    raise ValueError(f"Error: unsupported recipe {recipe_name}")
+                recipe = eval(
+                    f"{ATTACK_RECIPE_NAMES[recipe_name]}.build(model_wrapper, {params})"
+                )
+            elif args.attack_recipe in ATTACK_RECIPE_NAMES:
+                recipe = eval(
+                    f"{ATTACK_RECIPE_NAMES[args.attack_recipe]}.build(model_wrapper)"
+                )
+            else:
+                raise ValueError(f"Invalid recipe {args.attack_recipe}")
+            if args.query_budget:
+                recipe.goal_function.query_budget = args.query_budget
+            recipe.goal_function.model_cache_size = args.model_cache_size
+            recipe.constraint_cache_size = args.constraint_cache_size
+            return recipe
+        elif args.attack_from_file:
+            if ARGS_SPLIT_TOKEN in args.attack_from_file:
+                attack_file, attack_name = args.attack_from_file.split(ARGS_SPLIT_TOKEN)
+            else:
+                attack_file, attack_name = args.attack_from_file, "attack"
+            attack_module = load_module_from_file(attack_file)
+            if not hasattr(attack_module, attack_name):
+                raise ValueError(
+                    f"Loaded `{attack_file}` but could not find `{attack_name}`."
+                )
+            attack_func = getattr(attack_module, attack_name)
+            return attack_func(model_wrapper)
+        else:
+            goal_function = cls._create_goal_function_from_args(args, model_wrapper)
+            transformation = cls._create_transformation_from_args(args, model_wrapper)
+            constraints = cls._create_constraints_from_args(args)
+            if ARGS_SPLIT_TOKEN in args.search_method:
+                search_name, params = args.search_method.split(ARGS_SPLIT_TOKEN)
+                if search_name not in SEARCH_METHOD_CLASS_NAMES:
+                    raise ValueError(f"Error: unsupported search {search_name}")
+                search_method = eval(
+                    f"{SEARCH_METHOD_CLASS_NAMES[search_name]}({params})"
+                )
+            elif args.search_method in SEARCH_METHOD_CLASS_NAMES:
+                search_method = eval(
+                    f"{SEARCH_METHOD_CLASS_NAMES[args.search_method]}()"
+                )
+            else:
+                raise ValueError(f"Error: unsupported attack {args.search_method}")
+
+        return Attack(
+            goal_function,
+            constraints,
+            transformation,
+            search_method,
+            constraint_cache_size=args.constraint_cache_size,
+        )
+
+
+# This neat trick allows use to reorder the arguments to avoid TypeErrors commonly found when inheriting dataclass.
+# https://stackoverflow.com/questions/51575931/class-inheritance-in-python-3-7-dataclasses
+@dataclass
+class CommandLineAttackArgs(AttackArgs, _CommandLineAttackArgs, DatasetArgs, ModelArgs):
+    @classmethod
+    def _add_parser_args(cls, parser):
+        """Add listed args to command line parser."""
+        parser = ModelArgs._add_parser_args(parser)
+        parser = DatasetArgs._add_parser_args(parser)
+        parser = _CommandLineAttackArgs._add_parser_args(parser)
+        parser = AttackArgs._add_parser_args(parser)
+        return parser

textattack/attack_recipes/__init__.py

@@ -0,0 +1,43 @@
+""".. _attack_recipes:
+
+Attack Recipes Package:
+========================
+
+We provide a number of pre-built attack recipes, which correspond to attacks from the literature. To run an attack recipe from the command line, run::
+
+    textattack attack --recipe [recipe_name]
+
+To initialize an attack in Python script, use::
+
+    <recipe name>.build(model_wrapper)
+
+For example, ``attack = InputReductionFeng2018.build(model)`` creates `attack`, an object of type ``Attack`` with the goal function, transformation, constraints, and search method specified in that paper. This object can then be used just like any other attack; for example, by calling ``attack.attack_dataset``.
+
+TextAttack supports the following attack recipes (each recipe's documentation contains a link to the corresponding paper):
+
+.. contents:: :local:
+"""
+
+from .attack_recipe import AttackRecipe
+
+from .a2t_yoo_2021 import A2TYoo2021
+from .bae_garg_2019 import BAEGarg2019
+from .bert_attack_li_2020 import BERTAttackLi2020
+from .genetic_algorithm_alzantot_2018 import GeneticAlgorithmAlzantot2018
+from .faster_genetic_algorithm_jia_2019 import FasterGeneticAlgorithmJia2019
+from .deepwordbug_gao_2018 import DeepWordBugGao2018
+from .hotflip_ebrahimi_2017 import HotFlipEbrahimi2017
+from .input_reduction_feng_2018 import InputReductionFeng2018
+from .kuleshov_2017 import Kuleshov2017
+from .morpheus_tan_2020 import MorpheusTan2020
+from .seq2sick_cheng_2018_blackbox import Seq2SickCheng2018BlackBox
+from .textbugger_li_2018 import TextBuggerLi2018
+from .textfooler_jin_2019 import TextFoolerJin2019
+from .pwws_ren_2019 import PWWSRen2019
+from .iga_wang_2019 import IGAWang2019
+from .pruthi_2019 import Pruthi2019
+from .pso_zang_2020 import PSOZang2020
+from .checklist_ribeiro_2020 import CheckList2020
+from .clare_li_2020 import CLARE2020
+from .french_recipe import FrenchRecipe
+from .spanish_recipe import SpanishRecipe

textattack/attack_recipes/a2t_yoo_2021.py

@@ -0,0 +1,74 @@
+"""
+A2T (A2T: Attack for Adversarial Training Recipe)
+==================================================
+
+"""
+
+from textattack import Attack
+from textattack.constraints.grammaticality import PartOfSpeech
+from textattack.constraints.pre_transformation import (
+    InputColumnModification,
+    MaxModificationRate,
+    RepeatModification,
+    StopwordModification,
+)
+from textattack.constraints.semantics import WordEmbeddingDistance
+from textattack.constraints.semantics.sentence_encoders import BERT
+from textattack.goal_functions import UntargetedClassification
+from textattack.search_methods import GreedyWordSwapWIR
+from textattack.transformations import WordSwapEmbedding, WordSwapMaskedLM
+
+from .attack_recipe import AttackRecipe
+
+
+class A2TYoo2021(AttackRecipe):
+    """Towards Improving Adversarial Training of NLP Models.
+
+    (Yoo et al., 2021)
+
+    https://arxiv.org/abs/2109.00544
+    """
+
+    @staticmethod
+    def build(model_wrapper, mlm=False):
+        """Build attack recipe.
+
+        Args:
+            model_wrapper (:class:`~textattack.models.wrappers.ModelWrapper`):
+                Model wrapper containing both the model and the tokenizer.
+            mlm (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                If :obj:`True`, load `A2T-MLM` attack. Otherwise, load regular `A2T` attack.
+
+        Returns:
+            :class:`~textattack.Attack`: A2T attack.
+        """
+        constraints = [RepeatModification(), StopwordModification()]
+        input_column_modification = InputColumnModification(
+            ["premise", "hypothesis"], {"premise"}
+        )
+        constraints.append(input_column_modification)
+        constraints.append(PartOfSpeech(allow_verb_noun_swap=False))
+        constraints.append(MaxModificationRate(max_rate=0.1, min_threshold=4))
+        sent_encoder = BERT(
+            model_name="stsb-distilbert-base", threshold=0.9, metric="cosine"
+        )
+        constraints.append(sent_encoder)
+
+        if mlm:
+            transformation = transformation = WordSwapMaskedLM(
+                method="bae", max_candidates=20, min_confidence=0.0, batch_size=16
+            )
+        else:
+            transformation = WordSwapEmbedding(max_candidates=20)
+            constraints.append(WordEmbeddingDistance(min_cos_sim=0.8))
+
+        #
+        # Goal is untargeted classification
+        #
+        goal_function = UntargetedClassification(model_wrapper, model_batch_size=32)
+        #
+        # Greedily swap words with "Word Importance Ranking".
+        #
+        search_method = GreedyWordSwapWIR(wir_method="gradient")
+
+        return Attack(goal_function, constraints, transformation, search_method)

textattack/attack_recipes/attack_recipe.py

@@ -0,0 +1,30 @@
+"""
+Attack Recipe Class
+========================
+
+"""
+
+from abc import ABC, abstractmethod
+
+from textattack import Attack
+
+
+class AttackRecipe(Attack, ABC):
+    """A recipe for building an NLP adversarial attack from the literature."""
+
+    @staticmethod
+    @abstractmethod
+    def build(model_wrapper, **kwargs):
+        """Creates pre-built :class:`~textattack.Attack` that correspond to
+        attacks from the literature.
+
+        Args:
+            model_wrapper (:class:`~textattack.models.wrappers.ModelWrapper`):
+                :class:`~textattack.models.wrappers.ModelWrapper` that contains the victim model and tokenizer.
+                This is passed to :class:`~textattack.goal_functions.GoalFunction` when constructing the attack.
+            kwargs:
+                Additional keyword arguments.
+        Returns:
+            :class:`~textattack.Attack`
+        """
+        raise NotImplementedError()

textattack/attack_recipes/bae_garg_2019.py

@@ -0,0 +1,123 @@
+"""
+BAE (BAE: BERT-Based Adversarial Examples)
+============================================
+
+"""
+from textattack.constraints.grammaticality import PartOfSpeech
+from textattack.constraints.pre_transformation import (
+    RepeatModification,
+    StopwordModification,
+)
+from textattack.constraints.semantics.sentence_encoders import UniversalSentenceEncoder
+from textattack.goal_functions import UntargetedClassification
+from textattack.search_methods import GreedyWordSwapWIR
+from textattack.transformations import WordSwapMaskedLM
+
+from .attack_recipe import AttackRecipe
+
+
+class BAEGarg2019(AttackRecipe):
+    """Siddhant Garg and Goutham Ramakrishnan, 2019.
+
+    BAE: BERT-based Adversarial Examples for Text Classification.
+
+    https://arxiv.org/pdf/2004.01970
+
+    This is "attack mode" 1 from the paper, BAE-R, word replacement.
+
+    We present 4 attack modes for BAE based on the
+        R and I operations, where for each token t in S:
+        • BAE-R: Replace token t (See Algorithm 1)
+        • BAE-I: Insert a token to the left or right of t
+        • BAE-R/I: Either replace token t or insert a
+        token to the left or right of t
+        • BAE-R+I: First replace token t, then insert a
+        token to the left or right of t
+    """
+
+    @staticmethod
+    def build(model_wrapper):
+        # "In this paper, we present a simple yet novel technique: BAE (BERT-based
+        # Adversarial Examples), which uses a language model (LM) for token
+        # replacement to best fit the overall context. We perturb an input sentence
+        # by either replacing a token or inserting a new token in the sentence, by
+        # means of masking a part of the input and using a LM to fill in the mask."
+        #
+        # We only consider the top K=50 synonyms from the MLM predictions.
+        #
+        # [from email correspondance with the author]
+        # "When choosing the top-K candidates from the BERT masked LM, we filter out
+        # the sub-words and only retain the whole words (by checking if they are
+        # present in the GloVE vocabulary)"
+        #
+        transformation = WordSwapMaskedLM(
+            method="bae", max_candidates=50, min_confidence=0.0
+        )
+        #
+        # Don't modify the same word twice or stopwords.
+        #
+        constraints = [RepeatModification(), StopwordModification()]
+
+        # For the R operations we add an additional check for
+        # grammatical correctness of the generated adversarial example by filtering
+        # out predicted tokens that do not form the same part of speech (POS) as the
+        # original token t_i in the sentence.
+        constraints.append(PartOfSpeech(allow_verb_noun_swap=True))
+
+        # "To ensure semantic similarity on introducing perturbations in the input
+        # text, we filter the set of top-K masked tokens (K is a pre-defined
+        # constant) predicted by BERT-MLM using a Universal Sentence Encoder (USE)
+        # (Cer et al., 2018)-based sentence similarity scorer."
+        #
+        # "[We] set a threshold of 0.8 for the cosine similarity between USE-based
+        # embeddings of the adversarial and input text."
+        #
+        # [from email correspondence with the author]
+        # "For a fair comparison of the benefits of using a BERT-MLM in our paper,
+        # we retained the majority of TextFooler's specifications. Thus we:
+        # 1. Use the USE for comparison within a window of size 15 around the word
+        # being replaced/inserted.
+        # 2. Set the similarity score threshold to 0.1 for inputs shorter than the
+        # window size (this translates roughly to almost always accepting the new text).
+        # 3. Perform the USE similarity thresholding of 0.8 with respect to the text
+        # just before the replacement/insertion and not the original text (For
+        # example: at the 3rd R/I operation, we compute the USE score on a window
+        # of size 15 of the text obtained after the first 2 R/I operations and not
+        # the original text).
+        # ...
+        # To address point (3) from above, compare the USE with the original text
+        # at each iteration instead of the current one (While doing this change
+        # for the R-operation is trivial, doing it for the I-operation with the
+        # window based USE comparison might be more involved)."
+        #
+        # Finally, since the BAE code is based on the TextFooler code, we need to
+        # adjust the threshold to account for the missing / pi in the cosine
+        # similarity comparison. So the final threshold is 1 - (1 - 0.8) / pi
+        # = 1 - (0.2 / pi) = 0.936338023.
+        use_constraint = UniversalSentenceEncoder(
+            threshold=0.936338023,
+            metric="cosine",
+            compare_against_original=True,
+            window_size=15,
+            skip_text_shorter_than_window=True,
+        )
+        constraints.append(use_constraint)
+        #
+        # Goal is untargeted classification.
+        #
+        goal_function = UntargetedClassification(model_wrapper)
+        #
+        # "We estimate the token importance Ii of each token
+        # t_i ∈ S = [t1, . . . , tn], by deleting ti from S and computing the
+        # decrease in probability of predicting the correct label y, similar
+        # to (Jin et al., 2019).
+        #
+        # • "If there are multiple tokens can cause C to misclassify S when they
+        # replace the mask, we choose the token which makes Sadv most similar to
+        # the original S based on the USE score."
+        # • "If no token causes misclassification, we choose the perturbation that
+        # decreases the prediction probability P(C(Sadv)=y) the most."
+        #
+        search_method = GreedyWordSwapWIR(wir_method="delete")
+
+        return BAEGarg2019(goal_function, constraints, transformation, search_method)

textattack/attack_recipes/bert_attack_li_2020.py

@@ -0,0 +1,95 @@
+"""
+BERT-Attack:
+============================================================
+
+(BERT-Attack: Adversarial Attack Against BERT Using BERT)
+
+.. warning::
+    This attack is super slow
+    (see https://github.com/QData/TextAttack/issues/586)
+    Consider using smaller values for "max_candidates".
+
+"""
+from textattack import Attack
+from textattack.constraints.overlap import MaxWordsPerturbed
+from textattack.constraints.pre_transformation import (
+    RepeatModification,
+    StopwordModification,
+)
+from textattack.constraints.semantics.sentence_encoders import UniversalSentenceEncoder
+from textattack.goal_functions import UntargetedClassification
+from textattack.search_methods import GreedyWordSwapWIR
+from textattack.transformations import WordSwapMaskedLM
+
+from .attack_recipe import AttackRecipe
+
+
+class BERTAttackLi2020(AttackRecipe):
+    """Li, L.., Ma, R., Guo, Q., Xiangyang, X., Xipeng, Q. (2020).
+
+    BERT-ATTACK: Adversarial Attack Against BERT Using BERT
+
+    https://arxiv.org/abs/2004.09984
+
+    This is "attack mode" 1 from the paper, BAE-R, word replacement.
+    """
+
+    @staticmethod
+    def build(model_wrapper):
+        # [from correspondence with the author]
+        # Candidate size K is set to 48 for all data-sets.
+        transformation = WordSwapMaskedLM(method="bert-attack", max_candidates=48)
+        #
+        # Don't modify the same word twice or stopwords.
+        #
+        constraints = [RepeatModification(), StopwordModification()]
+
+        # "We only take ε percent of the most important words since we tend to keep
+        # perturbations minimum."
+        #
+        # [from correspondence with the author]
+        # "Word percentage allowed to change is set to 0.4 for most data-sets, this
+        # parameter is trivial since most attacks only need a few changes. This
+        # epsilon is only used to avoid too much queries on those very hard samples."
+        constraints.append(MaxWordsPerturbed(max_percent=0.4))
+
+        # "As used in TextFooler (Jin et al., 2019), we also use Universal Sentence
+        # Encoder (Cer et al., 2018) to measure the semantic consistency between the
+        # adversarial sample and the original sequence. To balance between semantic
+        # preservation and attack success rate, we set up a threshold of semantic
+        # similarity score to filter the less similar examples."
+        #
+        # [from correspondence with author]
+        # "Over the full texts, after generating all the adversarial samples, we filter
+        # out low USE score samples. Thus the success rate is lower but the USE score
+        # can be higher. (actually USE score is not a golden metric, so we simply
+        # measure the USE score over the final texts for a comparison with TextFooler).
+        # For datasets like IMDB, we set a higher threshold between 0.4-0.7; for
+        # datasets like MNLI, we set threshold between 0-0.2."
+        #
+        # Since the threshold in the real world can't be determined from the training
+        # data, the TextAttack implementation uses a fixed threshold - determined to
+        # be 0.2 to be most fair.
+        use_constraint = UniversalSentenceEncoder(
+            threshold=0.2,
+            metric="cosine",
+            compare_against_original=True,
+            window_size=None,
+        )
+        constraints.append(use_constraint)
+        #
+        # Goal is untargeted classification.
+        #
+        goal_function = UntargetedClassification(model_wrapper)
+        #
+        # "We first select the words in the sequence which have a high significance
+        # influence on the final output logit. Let S = [w0, ··· , wi ··· ] denote
+        # the input sentence, and oy(S) denote the logit output by the target model
+        # for correct label y, the importance score Iwi is defined as
+        # Iwi = oy(S) − oy(S\wi), where S\wi = [w0, ··· , wi−1, [MASK], wi+1, ···]
+        # is the sentence after replacing wi with [MASK]. Then we rank all the words
+        # according to the ranking score Iwi in descending order to create word list
+        # L."
+        search_method = GreedyWordSwapWIR(wir_method="unk")
+
+        return Attack(goal_function, constraints, transformation, search_method)

textattack/attack_recipes/checklist_ribeiro_2020.py

@@ -0,0 +1,53 @@
+"""
+CheckList:
+=========================
+
+(Beyond Accuracy: Behavioral Testing of NLP models with CheckList)
+
+"""
+from textattack import Attack
+from textattack.constraints.pre_transformation import RepeatModification
+from textattack.goal_functions import UntargetedClassification
+from textattack.search_methods import GreedySearch
+from textattack.transformations import (
+    CompositeTransformation,
+    WordSwapChangeLocation,
+    WordSwapChangeName,
+    WordSwapChangeNumber,
+    WordSwapContract,
+    WordSwapExtend,
+)
+
+from .attack_recipe import AttackRecipe
+
+
+class CheckList2020(AttackRecipe):
+    """An implementation of the attack used in "Beyond Accuracy: Behavioral
+    Testing of NLP models with CheckList", Ribeiro et al., 2020.
+
+    This attack focuses on a number of attacks used in the Invariance Testing
+    Method: Contraction, Extension, Changing Names, Number, Location
+
+    https://arxiv.org/abs/2005.04118
+    """
+
+    @staticmethod
+    def build(model_wrapper):
+        transformation = CompositeTransformation(
+            [
+                WordSwapExtend(),
+                WordSwapContract(),
+                WordSwapChangeName(),
+                WordSwapChangeNumber(),
+                WordSwapChangeLocation(),
+            ]
+        )
+
+        # Need this constraint to prevent extend and contract modifying each others' changes and forming infinite loop
+        constraints = [RepeatModification()]
+
+        # Untargeted attack & GreedySearch
+        goal_function = UntargetedClassification(model_wrapper)
+        search_method = GreedySearch()
+
+        return Attack(goal_function, constraints, transformation, search_method)