137 lines
4.8 KiB
Python
137 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Find and optionally delete duplicate files.
|
|
|
|
By default this identifies files that share the same size AND the same
|
|
content hash (i.e. true duplicates), keeps one copy of each, and only
|
|
*reports* what it would delete without touching anything. Pass --delete
|
|
to actually remove the extra copies.
|
|
|
|
WARNING: two files with the same *size* are not necessarily identical.
|
|
This script hashes same-sized files to confirm they are real duplicates
|
|
before deleting. Use --size-only to skip hashing and match purely on
|
|
size (dangerous: this can delete unrelated files).
|
|
|
|
Examples:
|
|
python dedupe.py ~/Downloads # dry run, content-verified
|
|
python dedupe.py ~/Downloads --delete # actually delete duplicates
|
|
python dedupe.py ~/Downloads --size-only # match on size only (risky)
|
|
python dedupe.py ~/Downloads --no-recursive # top level only
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import sys
|
|
from collections import defaultdict
|
|
|
|
|
|
def iter_files(root, recursive=True):
|
|
if recursive:
|
|
for dirpath, _, filenames in os.walk(root):
|
|
for name in filenames:
|
|
path = os.path.join(dirpath, name)
|
|
if os.path.isfile(path) and not os.path.islink(path):
|
|
yield path
|
|
else:
|
|
for name in os.listdir(root):
|
|
path = os.path.join(root, name)
|
|
if os.path.isfile(path) and not os.path.islink(path):
|
|
yield path
|
|
|
|
|
|
def hash_file(path, chunk_size=1 << 20):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(chunk_size), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def group_by_size(paths):
|
|
sizes = defaultdict(list)
|
|
for path in paths:
|
|
try:
|
|
sizes[os.path.getsize(path)].append(path)
|
|
except OSError as e:
|
|
print(f"skip (cannot stat): {path} ({e})", file=sys.stderr)
|
|
return sizes
|
|
|
|
|
|
def find_duplicates(root, recursive=True, size_only=False):
|
|
"""Yield lists of files that are duplicates of one another."""
|
|
size_groups = group_by_size(iter_files(root, recursive))
|
|
for size, paths in size_groups.items():
|
|
if len(paths) < 2:
|
|
continue
|
|
if size_only:
|
|
yield paths
|
|
continue
|
|
# Confirm real duplicates by content hash.
|
|
hashes = defaultdict(list)
|
|
for path in paths:
|
|
try:
|
|
hashes[hash_file(path)].append(path)
|
|
except OSError as e:
|
|
print(f"skip (cannot read): {path} ({e})", file=sys.stderr)
|
|
for same in hashes.values():
|
|
if len(same) > 1:
|
|
yield same
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Find/delete duplicate files.")
|
|
parser.add_argument("directory", help="Directory to scan")
|
|
parser.add_argument("--delete", action="store_true",
|
|
help="Actually delete duplicates (default: dry run)")
|
|
parser.add_argument("--size-only", action="store_true",
|
|
help="Match on size only, skip content hashing (DANGEROUS)")
|
|
parser.add_argument("--no-recursive", action="store_true",
|
|
help="Do not descend into subdirectories")
|
|
args = parser.parse_args()
|
|
|
|
if not os.path.isdir(args.directory):
|
|
parser.error(f"not a directory: {args.directory}")
|
|
|
|
if args.size_only and args.delete:
|
|
print("WARNING: --size-only with --delete removes files based on size "
|
|
"alone. Same size does NOT mean same content.\n", file=sys.stderr)
|
|
|
|
total_removed = 0
|
|
total_reclaimed = 0
|
|
|
|
for group in find_duplicates(args.directory,
|
|
recursive=not args.no_recursive,
|
|
size_only=args.size_only):
|
|
group.sort() # deterministic: keep the first path alphabetically
|
|
keep, remove = group[0], group[1:]
|
|
print(f"\nDuplicate set ({len(group)} files):")
|
|
print(f" keep: {keep}")
|
|
for path in remove:
|
|
try:
|
|
size = os.path.getsize(path)
|
|
except OSError:
|
|
size = 0
|
|
if args.delete:
|
|
try:
|
|
os.remove(path)
|
|
print(f" DELETED {path}")
|
|
total_removed += 1
|
|
total_reclaimed += size
|
|
except OSError as e:
|
|
print(f" ERROR deleting {path}: {e}", file=sys.stderr)
|
|
else:
|
|
print(f" would delete: {path}")
|
|
total_removed += 1
|
|
total_reclaimed += size
|
|
|
|
verb = "Deleted" if args.delete else "Would delete"
|
|
print(f"\n{verb} {total_removed} file(s), "
|
|
f"reclaiming {total_reclaimed / (1024 * 1024):.2f} MiB.")
|
|
if not args.delete and total_removed:
|
|
print("Re-run with --delete to actually remove them.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|