منظف تكرار CSV

import argparse
import csv
from pathlib import Path


def deduplicate(input_path: Path, output_path: Path, key: str) -> tuple[int, int]:
    seen = set()
    kept = 0
    skipped = 0
    with input_path.open(newline="", encoding="utf-8") as source:
        reader = csv.DictReader(source)
        if not reader.fieldnames or key not in reader.fieldnames:
            raise SystemExit(f"Missing CSV column: {key}")
        with output_path.open("w", newline="", encoding="utf-8") as target:
            writer = csv.DictWriter(target, fieldnames=reader.fieldnames)
            writer.writeheader()
            for row in reader:
                marker = row.get(key, "")
                if marker in seen:
                    skipped += 1
                    continue
                seen.add(marker)
                writer.writerow(row)
                kept += 1
    return kept, skipped


def main() -> None:
    parser = argparse.ArgumentParser(description="Write a deduplicated CSV copy by key column.")
    parser.add_argument("input", type=Path)
    parser.add_argument("output", type=Path)
    parser.add_argument("--key", required=True)
    args = parser.parse_args()
    kept, skipped = deduplicate(args.input, args.output, args.key)
    print(f"Kept {kept} rows, skipped {skipped} duplicates")


if __name__ == "__main__":
    main()