Charles Kabui
commited on
Commit
·
8404c62
1
Parent(s):
1843265
unzip_fn
Browse files
utils/remove_duplicates.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1 |
-
def
|
2 |
return [list(i) for i in zip(*items)]
|
3 |
|
4 |
def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=False):
|
@@ -16,4 +16,4 @@ def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=Fa
|
|
16 |
import tqdm
|
17 |
progress = tqdm.tqdm
|
18 |
deduped_items = list({key(item): item for item in progress(items, desc='Deduping...')}.values())
|
19 |
-
return deduped_items if not unzip else
|
|
|
1 |
+
def unzip_fn(items: list):
|
2 |
return [list(i) for i in zip(*items)]
|
3 |
|
4 |
def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=False):
|
|
|
16 |
import tqdm
|
17 |
progress = tqdm.tqdm
|
18 |
deduped_items = list({key(item): item for item in progress(items, desc='Deduping...')}.values())
|
19 |
+
return deduped_items if not unzip else unzip_fn(deduped_items)
|