Charles Kabui commited on
Commit
8404c62
·
1 Parent(s): 1843265
Files changed (1) hide show
  1. utils/remove_duplicates.py +2 -2
utils/remove_duplicates.py CHANGED
@@ -1,4 +1,4 @@
1
- def unzip(items: list):
2
  return [list(i) for i in zip(*items)]
3
 
4
  def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=False):
@@ -16,4 +16,4 @@ def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=Fa
16
  import tqdm
17
  progress = tqdm.tqdm
18
  deduped_items = list({key(item): item for item in progress(items, desc='Deduping...')}.values())
19
- return deduped_items if not unzip else unzip(deduped_items)
 
1
+ def unzip_fn(items: list):
2
  return [list(i) for i in zip(*items)]
3
 
4
  def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=False):
 
16
  import tqdm
17
  progress = tqdm.tqdm
18
  deduped_items = list({key(item): item for item in progress(items, desc='Deduping...')}.values())
19
+ return deduped_items if not unzip else unzip_fn(deduped_items)