v1

dedupe.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""
+Find and optionally delete duplicate files.
+
+By default this identifies files that share the same size AND the same
+content hash (i.e. true duplicates), keeps one copy of each, and only
+*reports* what it would delete without touching anything. Pass --delete
+to actually remove the extra copies.
+
+WARNING: two files with the same *size* are not necessarily identical.
+This script hashes same-sized files to confirm they are real duplicates
+before deleting. Use --size-only to skip hashing and match purely on
+size (dangerous: this can delete unrelated files).
+
+Examples:
+    python dedupe.py ~/Downloads                 # dry run, content-verified
+    python dedupe.py ~/Downloads --delete        # actually delete duplicates
+    python dedupe.py ~/Downloads --size-only     # match on size only (risky)
+    python dedupe.py ~/Downloads --no-recursive  # top level only
+"""
+
+import argparse
+import hashlib
+import os
+import sys
+from collections import defaultdict
+
+
+def iter_files(root, recursive=True):
+    if recursive:
+        for dirpath, _, filenames in os.walk(root):
+            for name in filenames:
+                path = os.path.join(dirpath, name)
+                if os.path.isfile(path) and not os.path.islink(path):
+                    yield path
+    else:
+        for name in os.listdir(root):
+            path = os.path.join(root, name)
+            if os.path.isfile(path) and not os.path.islink(path):
+                yield path
+
+
+def hash_file(path, chunk_size=1 << 20):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(chunk_size), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def group_by_size(paths):
+    sizes = defaultdict(list)
+    for path in paths:
+        try:
+            sizes[os.path.getsize(path)].append(path)
+        except OSError as e:
+            print(f"skip (cannot stat): {path} ({e})", file=sys.stderr)
+    return sizes
+
+
+def find_duplicates(root, recursive=True, size_only=False):
+    """Yield lists of files that are duplicates of one another."""
+    size_groups = group_by_size(iter_files(root, recursive))
+    for size, paths in size_groups.items():
+        if len(paths) < 2:
+            continue
+        if size_only:
+            yield paths
+            continue
+        # Confirm real duplicates by content hash.
+        hashes = defaultdict(list)
+        for path in paths:
+            try:
+                hashes[hash_file(path)].append(path)
+            except OSError as e:
+                print(f"skip (cannot read): {path} ({e})", file=sys.stderr)
+        for same in hashes.values():
+            if len(same) > 1:
+                yield same
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Find/delete duplicate files.")
+    parser.add_argument("directory", help="Directory to scan")
+    parser.add_argument("--delete", action="store_true",
+                        help="Actually delete duplicates (default: dry run)")
+    parser.add_argument("--size-only", action="store_true",
+                        help="Match on size only, skip content hashing (DANGEROUS)")
+    parser.add_argument("--no-recursive", action="store_true",
+                        help="Do not descend into subdirectories")
+    args = parser.parse_args()
+
+    if not os.path.isdir(args.directory):
+        parser.error(f"not a directory: {args.directory}")
+
+    if args.size_only and args.delete:
+        print("WARNING: --size-only with --delete removes files based on size "
+              "alone. Same size does NOT mean same content.\n", file=sys.stderr)
+
+    total_removed = 0
+    total_reclaimed = 0
+
+    for group in find_duplicates(args.directory,
+                                 recursive=not args.no_recursive,
+                                 size_only=args.size_only):
+        group.sort()  # deterministic: keep the first path alphabetically
+        keep, remove = group[0], group[1:]
+        print(f"\nDuplicate set ({len(group)} files):")
+        print(f"  keep:   {keep}")
+        for path in remove:
+            try:
+                size = os.path.getsize(path)
+            except OSError:
+                size = 0
+            if args.delete:
+                try:
+                    os.remove(path)
+                    print(f"  DELETED {path}")
+                    total_removed += 1
+                    total_reclaimed += size
+                except OSError as e:
+                    print(f"  ERROR deleting {path}: {e}", file=sys.stderr)
+            else:
+                print(f"  would delete: {path}")
+                total_removed += 1
+                total_reclaimed += size
+
+    verb = "Deleted" if args.delete else "Would delete"
+    print(f"\n{verb} {total_removed} file(s), "
+          f"reclaiming {total_reclaimed / (1024 * 1024):.2f} MiB.")
+    if not args.delete and total_removed:
+        print("Re-run with --delete to actually remove them.")
+
+
+if __name__ == "__main__":
+    main()