#!/usr/bin/env python3
"""Audit ZIP directory without fetching the 15 GB audio archive."""
import collections
import hashlib
import io
import json
from pathlib import Path
import re
import time
import urllib.request
import zipfile

ROOT = Path(__file__).resolve().parents[1]
CACHE = Path('/mnt/data/research/bioinformatics-discovery/cache')
URL = 'https://zenodo.org/api/records/17573913/files/zenodo_nbm_db.zip/content'
SIZE = 14982302793

class RangeFile(io.RawIOBase):
    def __init__(self):
        self.pos = 0
        self.transferred = 0
    def seekable(self): return True
    def readable(self): return True
    def tell(self): return self.pos
    def seek(self, offset, whence=0):
        self.pos = offset if whence == 0 else self.pos + offset if whence == 1 else SIZE + offset
        return self.pos
    def read(self, size=-1):
        if size < 0: size = SIZE - self.pos
        if size == 0: return b''
        if size > 2_000_000: raise ValueError('Refusing an unexpectedly large range')
        end = min(SIZE - 1, self.pos + size - 1)
        path = CACHE / f'nbm-range-{self.pos}-{end}.bin'
        if path.exists(): data = path.read_bytes()
        else:
            req = urllib.request.Request(URL, headers={'Range':f'bytes={self.pos}-{end}'})
            with urllib.request.urlopen(req, timeout=45) as res:
                expected = f'bytes {self.pos}-{end}/{SIZE}'
                if res.status != 206 or res.headers.get('Content-Range') != expected:
                    raise ValueError(f'Range unsupported or wrong: {res.status}, {res.headers.get("Content-Range")}')
                data = res.read(size + 1)
            if len(data) != end - self.pos + 1: raise ValueError('Wrong byte count')
            path.write_bytes(data)
            self.transferred += len(data)
            time.sleep(.15)
        self.pos += len(data)
        return data

def main():
    source = RangeFile()
    with zipfile.ZipFile(source) as z:
        rows = [dict(name=i.filename, size=i.file_size, compressed=i.compress_size,
                     crc32=i.CRC, offset=i.header_offset) for i in z.infolist() if not i.is_dir()]
    (ROOT/'evidence/nbm_archive_directory.json').write_text(json.dumps(rows,indent=2))
    audio = [r for r in rows if r['name'].endswith('.wav')]
    def split(name): return 'test' if '/test/' in '/'+name else 'train'
    groups = collections.defaultdict(list)
    ids = collections.defaultdict(set)
    for r in audio:
        groups[(r['size'],r['crc32'])].append(r['name'])
        match = re.search(r'#(\d+)',r['name'])
        if match: ids[match[1]].add(split(r['name']))
    dupes = [v for v in groups.values() if len(v)>1]
    results = dict(audio_count=len(audio),split_counts=dict(collections.Counter(split(r['name']) for r in audio)),
        shared_xc_ids=[k for k,v in ids.items() if len(v)>1],
        duplicate_size_crc_candidates=dupes,
        cross_split_crc_candidates=[v for v in dupes if len({split(x) for x in v})>1],
        bytes_downloaded_this_run=source.transferred,
        note='CRC32 plus byte length screens exact file duplicates, not cropped/re-encoded or same-session recordings. Verify any candidates with cryptographic hashes before asserting identity.')
    (ROOT/'results/001_nbm_source_audit.json').write_text(json.dumps(results,indent=2))
    print(json.dumps(results,indent=2))

if __name__ == '__main__': main()