RE_UPLOAD-REBUILD-RESTART

utils/remove_duplicates.py

@@ -0,0 +1,19 @@
+def unzip_fn(items: list):
+    return [list(i) for i in zip(*items)]
+
+def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=False):
+    '''
+    Remove duplicates from a list of items
+    Args:
+      items: List of items
+      key: Function to get the key of the item
+      show_process: Whether to show the process or not
+    Returns:
+      List: List of items without duplicates
+    '''
+    progress = lambda x, *, desc: x
+    if show_process:
+        import tqdm
+        progress = tqdm.tqdm
+    deduped_items =  list({key(item): item for item in progress(items, desc='Deduping...')}.values())
+    return deduped_items if not unzip else unzip_fn(deduped_items)