Fix unsupported operand type(s) for |
Browse files
    	
        src/axolotl/prompt_tokenizers.py
    CHANGED
    
    | @@ -4,7 +4,7 @@ import abc | |
| 4 | 
             
            import copy
         | 
| 5 | 
             
            import functools
         | 
| 6 | 
             
            import logging
         | 
| 7 | 
            -
            from typing import Dict, List, Tuple
         | 
| 8 |  | 
| 9 | 
             
            from transformers import PreTrainedTokenizer
         | 
| 10 |  | 
| @@ -433,7 +433,7 @@ def parse_tokenized_to_result( | |
| 433 | 
             
                current_len: int,
         | 
| 434 | 
             
                res: Dict[str, List[int]],
         | 
| 435 | 
             
                labels: list[int],
         | 
| 436 | 
            -
                pad_token_id: int  | 
| 437 | 
             
            ) -> Tuple[Dict[str, List[int]], int]:
         | 
| 438 | 
             
                """
         | 
| 439 | 
             
                Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result
         | 
|  | |
| 4 | 
             
            import copy
         | 
| 5 | 
             
            import functools
         | 
| 6 | 
             
            import logging
         | 
| 7 | 
            +
            from typing import Dict, List, Tuple, Union
         | 
| 8 |  | 
| 9 | 
             
            from transformers import PreTrainedTokenizer
         | 
| 10 |  | 
|  | |
| 433 | 
             
                current_len: int,
         | 
| 434 | 
             
                res: Dict[str, List[int]],
         | 
| 435 | 
             
                labels: list[int],
         | 
| 436 | 
            +
                pad_token_id: Union[int, None] = None,
         | 
| 437 | 
             
            ) -> Tuple[Dict[str, List[int]], int]:
         | 
| 438 | 
             
                """
         | 
| 439 | 
             
                Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result
         |