From cdd4bc1caf1fbec18473492aeeb017bbdb79847e Mon Sep 17 00:00:00 2001 From: jpmvaz Date: Sun, 13 Sep 2026 19:51:51 +0100 Subject: [PATCH] v1 --- README.md | 65 ++++++++++++++++++++++++++ dedupe.py | 136 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 README.md create mode 100644 dedupe.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..7422c75 --- /dev/null +++ b/README.md @@ -0,0 +1,65 @@ +# dedupe.py — Duplicate File Finder / Deleter + +Scans a directory for duplicate files, keeps one copy of each, and reports or deletes the rest. + +## What it does + +It groups files by size as a fast first pass, then confirms real duplicates by comparing **SHA-256 content hashes**. For each set of identical files it keeps one copy and reports (or deletes) the others. + +By default the script runs in **dry-run mode** — it only tells you what it *would* delete. Nothing is removed unless you explicitly pass `--delete`. + +## ⚠️ Important + +Two files with the same **size** are *not* necessarily the same file. That's why the script hashes file contents before deleting. Only use size-only matching (`--size-only`) if you fully understand the risk of deleting unrelated files. + +## Requirements + +- Python 3.6 or newer +- No third-party packages (standard library only) + +## Usage + +``` +python dedupe.py DIRECTORY [options] +``` + +`DIRECTORY` is the folder to scan. + +### Options + +| Option | Description | +| --- | --- | +| `--delete` | Actually delete the duplicate copies. Without this flag the script only prints what it would do (dry run). | +| `--size-only` | Match files on size alone and skip content hashing. **Dangerous:** same size does not mean same content. | +| `--no-recursive` | Only scan the top level of `DIRECTORY`; do not descend into subfolders. | +| `-h`, `--help` | Show the built-in help message. | + +### Examples + +```bash +# See what would be deleted in ~/Downloads (safe, nothing removed) +python dedupe.py ~/Downloads + +# Actually delete verified duplicates +python dedupe.py ~/Downloads --delete + +# Only look at the top level, not subfolders +python dedupe.py ~/Downloads --no-recursive + +# Match purely on size (risky) — dry run first +python dedupe.py ~/Downloads --size-only +``` + +## How it decides what to keep + +Within each set of duplicates, the files are sorted alphabetically by path and the **first** one is kept. All others in the set are the candidates for deletion. + +## Output + +For each duplicate set the script prints the file it keeps and each file it would delete (or has deleted). At the end it prints a summary with the number of files affected and the amount of disk space reclaimed. + +## Safety tips + +1. Always run once **without** `--delete` and review the output first. +2. Make sure you have a backup before deleting anything important. +3. Avoid `--size-only` unless you're certain size alone is enough. diff --git a/dedupe.py b/dedupe.py new file mode 100644 index 0000000..17e432a --- /dev/null +++ b/dedupe.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +""" +Find and optionally delete duplicate files. + +By default this identifies files that share the same size AND the same +content hash (i.e. true duplicates), keeps one copy of each, and only +*reports* what it would delete without touching anything. Pass --delete +to actually remove the extra copies. + +WARNING: two files with the same *size* are not necessarily identical. +This script hashes same-sized files to confirm they are real duplicates +before deleting. Use --size-only to skip hashing and match purely on +size (dangerous: this can delete unrelated files). + +Examples: + python dedupe.py ~/Downloads # dry run, content-verified + python dedupe.py ~/Downloads --delete # actually delete duplicates + python dedupe.py ~/Downloads --size-only # match on size only (risky) + python dedupe.py ~/Downloads --no-recursive # top level only +""" + +import argparse +import hashlib +import os +import sys +from collections import defaultdict + + +def iter_files(root, recursive=True): + if recursive: + for dirpath, _, filenames in os.walk(root): + for name in filenames: + path = os.path.join(dirpath, name) + if os.path.isfile(path) and not os.path.islink(path): + yield path + else: + for name in os.listdir(root): + path = os.path.join(root, name) + if os.path.isfile(path) and not os.path.islink(path): + yield path + + +def hash_file(path, chunk_size=1 << 20): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + h.update(chunk) + return h.hexdigest() + + +def group_by_size(paths): + sizes = defaultdict(list) + for path in paths: + try: + sizes[os.path.getsize(path)].append(path) + except OSError as e: + print(f"skip (cannot stat): {path} ({e})", file=sys.stderr) + return sizes + + +def find_duplicates(root, recursive=True, size_only=False): + """Yield lists of files that are duplicates of one another.""" + size_groups = group_by_size(iter_files(root, recursive)) + for size, paths in size_groups.items(): + if len(paths) < 2: + continue + if size_only: + yield paths + continue + # Confirm real duplicates by content hash. + hashes = defaultdict(list) + for path in paths: + try: + hashes[hash_file(path)].append(path) + except OSError as e: + print(f"skip (cannot read): {path} ({e})", file=sys.stderr) + for same in hashes.values(): + if len(same) > 1: + yield same + + +def main(): + parser = argparse.ArgumentParser(description="Find/delete duplicate files.") + parser.add_argument("directory", help="Directory to scan") + parser.add_argument("--delete", action="store_true", + help="Actually delete duplicates (default: dry run)") + parser.add_argument("--size-only", action="store_true", + help="Match on size only, skip content hashing (DANGEROUS)") + parser.add_argument("--no-recursive", action="store_true", + help="Do not descend into subdirectories") + args = parser.parse_args() + + if not os.path.isdir(args.directory): + parser.error(f"not a directory: {args.directory}") + + if args.size_only and args.delete: + print("WARNING: --size-only with --delete removes files based on size " + "alone. Same size does NOT mean same content.\n", file=sys.stderr) + + total_removed = 0 + total_reclaimed = 0 + + for group in find_duplicates(args.directory, + recursive=not args.no_recursive, + size_only=args.size_only): + group.sort() # deterministic: keep the first path alphabetically + keep, remove = group[0], group[1:] + print(f"\nDuplicate set ({len(group)} files):") + print(f" keep: {keep}") + for path in remove: + try: + size = os.path.getsize(path) + except OSError: + size = 0 + if args.delete: + try: + os.remove(path) + print(f" DELETED {path}") + total_removed += 1 + total_reclaimed += size + except OSError as e: + print(f" ERROR deleting {path}: {e}", file=sys.stderr) + else: + print(f" would delete: {path}") + total_removed += 1 + total_reclaimed += size + + verb = "Deleted" if args.delete else "Would delete" + print(f"\n{verb} {total_removed} file(s), " + f"reclaiming {total_reclaimed / (1024 * 1024):.2f} MiB.") + if not args.delete and total_removed: + print("Re-run with --delete to actually remove them.") + + +if __name__ == "__main__": + main()