v1
This commit is contained in:
jpmvaz
@@ -0,0 +1,65 @@
|
||||
# dedupe.py — Duplicate File Finder / Deleter
|
||||
|
||||
Scans a directory for duplicate files, keeps one copy of each, and reports or deletes the rest.
|
||||
|
||||
## What it does
|
||||
|
||||
It groups files by size as a fast first pass, then confirms real duplicates by comparing **SHA-256 content hashes**. For each set of identical files it keeps one copy and reports (or deletes) the others.
|
||||
|
||||
By default the script runs in **dry-run mode** — it only tells you what it *would* delete. Nothing is removed unless you explicitly pass `--delete`.
|
||||
|
||||
## ⚠️ Important
|
||||
|
||||
Two files with the same **size** are *not* necessarily the same file. That's why the script hashes file contents before deleting. Only use size-only matching (`--size-only`) if you fully understand the risk of deleting unrelated files.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.6 or newer
|
||||
- No third-party packages (standard library only)
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
python dedupe.py DIRECTORY [options]
|
||||
```
|
||||
|
||||
`DIRECTORY` is the folder to scan.
|
||||
|
||||
### Options
|
||||
|
||||
| Option | Description |
|
||||
| --- | --- |
|
||||
| `--delete` | Actually delete the duplicate copies. Without this flag the script only prints what it would do (dry run). |
|
||||
| `--size-only` | Match files on size alone and skip content hashing. **Dangerous:** same size does not mean same content. |
|
||||
| `--no-recursive` | Only scan the top level of `DIRECTORY`; do not descend into subfolders. |
|
||||
| `-h`, `--help` | Show the built-in help message. |
|
||||
|
||||
### Examples
|
||||
|
||||
```bash
|
||||
# See what would be deleted in ~/Downloads (safe, nothing removed)
|
||||
python dedupe.py ~/Downloads
|
||||
|
||||
# Actually delete verified duplicates
|
||||
python dedupe.py ~/Downloads --delete
|
||||
|
||||
# Only look at the top level, not subfolders
|
||||
python dedupe.py ~/Downloads --no-recursive
|
||||
|
||||
# Match purely on size (risky) — dry run first
|
||||
python dedupe.py ~/Downloads --size-only
|
||||
```
|
||||
|
||||
## How it decides what to keep
|
||||
|
||||
Within each set of duplicates, the files are sorted alphabetically by path and the **first** one is kept. All others in the set are the candidates for deletion.
|
||||
|
||||
## Output
|
||||
|
||||
For each duplicate set the script prints the file it keeps and each file it would delete (or has deleted). At the end it prints a summary with the number of files affected and the amount of disk space reclaimed.
|
||||
|
||||
## Safety tips
|
||||
|
||||
1. Always run once **without** `--delete` and review the output first.
|
||||
2. Make sure you have a backup before deleting anything important.
|
||||
3. Avoid `--size-only` unless you're certain size alone is enough.
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Find and optionally delete duplicate files.
|
||||
|
||||
By default this identifies files that share the same size AND the same
|
||||
content hash (i.e. true duplicates), keeps one copy of each, and only
|
||||
*reports* what it would delete without touching anything. Pass --delete
|
||||
to actually remove the extra copies.
|
||||
|
||||
WARNING: two files with the same *size* are not necessarily identical.
|
||||
This script hashes same-sized files to confirm they are real duplicates
|
||||
before deleting. Use --size-only to skip hashing and match purely on
|
||||
size (dangerous: this can delete unrelated files).
|
||||
|
||||
Examples:
|
||||
python dedupe.py ~/Downloads # dry run, content-verified
|
||||
python dedupe.py ~/Downloads --delete # actually delete duplicates
|
||||
python dedupe.py ~/Downloads --size-only # match on size only (risky)
|
||||
python dedupe.py ~/Downloads --no-recursive # top level only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
def iter_files(root, recursive=True):
|
||||
if recursive:
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
for name in filenames:
|
||||
path = os.path.join(dirpath, name)
|
||||
if os.path.isfile(path) and not os.path.islink(path):
|
||||
yield path
|
||||
else:
|
||||
for name in os.listdir(root):
|
||||
path = os.path.join(root, name)
|
||||
if os.path.isfile(path) and not os.path.islink(path):
|
||||
yield path
|
||||
|
||||
|
||||
def hash_file(path, chunk_size=1 << 20):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(chunk_size), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def group_by_size(paths):
|
||||
sizes = defaultdict(list)
|
||||
for path in paths:
|
||||
try:
|
||||
sizes[os.path.getsize(path)].append(path)
|
||||
except OSError as e:
|
||||
print(f"skip (cannot stat): {path} ({e})", file=sys.stderr)
|
||||
return sizes
|
||||
|
||||
|
||||
def find_duplicates(root, recursive=True, size_only=False):
|
||||
"""Yield lists of files that are duplicates of one another."""
|
||||
size_groups = group_by_size(iter_files(root, recursive))
|
||||
for size, paths in size_groups.items():
|
||||
if len(paths) < 2:
|
||||
continue
|
||||
if size_only:
|
||||
yield paths
|
||||
continue
|
||||
# Confirm real duplicates by content hash.
|
||||
hashes = defaultdict(list)
|
||||
for path in paths:
|
||||
try:
|
||||
hashes[hash_file(path)].append(path)
|
||||
except OSError as e:
|
||||
print(f"skip (cannot read): {path} ({e})", file=sys.stderr)
|
||||
for same in hashes.values():
|
||||
if len(same) > 1:
|
||||
yield same
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Find/delete duplicate files.")
|
||||
parser.add_argument("directory", help="Directory to scan")
|
||||
parser.add_argument("--delete", action="store_true",
|
||||
help="Actually delete duplicates (default: dry run)")
|
||||
parser.add_argument("--size-only", action="store_true",
|
||||
help="Match on size only, skip content hashing (DANGEROUS)")
|
||||
parser.add_argument("--no-recursive", action="store_true",
|
||||
help="Do not descend into subdirectories")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isdir(args.directory):
|
||||
parser.error(f"not a directory: {args.directory}")
|
||||
|
||||
if args.size_only and args.delete:
|
||||
print("WARNING: --size-only with --delete removes files based on size "
|
||||
"alone. Same size does NOT mean same content.\n", file=sys.stderr)
|
||||
|
||||
total_removed = 0
|
||||
total_reclaimed = 0
|
||||
|
||||
for group in find_duplicates(args.directory,
|
||||
recursive=not args.no_recursive,
|
||||
size_only=args.size_only):
|
||||
group.sort() # deterministic: keep the first path alphabetically
|
||||
keep, remove = group[0], group[1:]
|
||||
print(f"\nDuplicate set ({len(group)} files):")
|
||||
print(f" keep: {keep}")
|
||||
for path in remove:
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
except OSError:
|
||||
size = 0
|
||||
if args.delete:
|
||||
try:
|
||||
os.remove(path)
|
||||
print(f" DELETED {path}")
|
||||
total_removed += 1
|
||||
total_reclaimed += size
|
||||
except OSError as e:
|
||||
print(f" ERROR deleting {path}: {e}", file=sys.stderr)
|
||||
else:
|
||||
print(f" would delete: {path}")
|
||||
total_removed += 1
|
||||
total_reclaimed += size
|
||||
|
||||
verb = "Deleted" if args.delete else "Would delete"
|
||||
print(f"\n{verb} {total_removed} file(s), "
|
||||
f"reclaiming {total_reclaimed / (1024 * 1024):.2f} MiB.")
|
||||
if not args.delete and total_removed:
|
||||
print("Re-run with --delete to actually remove them.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user