def remove_duplicates(items: list, key=lambda x: x, show_process=False): ''' Remove duplicates from a list of items Args: items: List of items key: Function to get the key of the item show_process: Whether to show the process or not Returns: List: List of items without duplicates ''' progress = lambda x, *, desc: x if show_process: import tqdm progress = tqdm.tqdm return list({key(item): item for item in progress(items, desc='Deduping...')}.values())