Upload processor

processor.py

@@ -13,7 +13,7 @@ class GIAProcessor(GitProcessor):
         }
         for i in range(len(examples["input_ids"])):
             _input_size = len(examples["input_ids"][i])
-            for j in range(max(1, _input_size // max_input_size)):
+            for j in range(max(1, _input_size // max_input_size)):  # skip last if smaller than max_input_size
                 results["input_ids"].append(examples["input_ids"][i][j*max_input_size:(j + 1) * max_input_size])
                 results["attention_mask"].append(examples["attention_mask"][i][j * max_input_size:(j + 1) * max_input_size])
 
@@ -21,7 +21,8 @@ class GIAProcessor(GitProcessor):
 
     def __call__(self, examples, max_input_size, return_tensors=None, **kwargs):
         if "text" in examples and not "images" in examples:
-            encoded_text = self.tokenizer(examples["text"], return_tensors=return_tensors)
+            encoded_text = self.tokenizer(examples["text"], return_tensors=return_tensors, max_length=max_input_size,
+                                          truncation=False, padding="max_length")
             encoding = self._cut_text(encoded_text, max_input_size)
         elif "text" in examples and "images" in examples:
             encoding = super().__call__(examples["text"], examples["images"], return_tensors, **kwargs)