"""T-SEG-001: f1v layout-preserving transcription audit, not decipherment.

Run from any directory with Python 3; inputs are the unmodified IT/ZL files.
The deliberately broken path is only a regression control, never a reading.
"""
from pathlib import Path
import hashlib
import json
import re

ROOT = Path(__file__).resolve().parent

def normalize(segment, split):
    segment = re.sub(r'<[^>]*>', '', segment)
    segment = re.sub(r'\[([^\[\]]+)\]', lambda m: m[1].split(':')[0], segment)
    segment = segment.replace('{', '').replace('}', '')
    segment = segment.replace(',', '.' if split else '')
    return [t for t in segment.split('.') if t]

def main():
    result = {
        'test_id': 'T-SEG-001', 'date': '2026-09-17', 'folio': 'f1v',
        'status': 'EXECUTED: descriptive segmentation and regression control',
        'normalization': 'Split each raw locus at <-> before removing metadata tags. Keep left/right segments separate. Periods split tokens. Run comma uncertainty joined and split. Select first bracketed alternative only for counts; retain raw alternatives. No joining across a drawing or line boundary.',
        'limitations': ['Tokens are operational transcription units, not proven words.', 'IT and ZL are related scholarly transcriptions, not independent manuscript witnesses.', 'No glyph-by-glyph adjudication or language model.', 'No significance test; no Latin conversion, medical meaning, cipher key, or plaintext established.', 'Cross-drawing continuity of reading is an editorial assumption, not deciphered syntax.'],
        'sources': {}, 'runs': {}, 'regression_checks': {},
    }
    for edition, filename in [('IT', 'IT2a-n.txt'), ('ZL', 'ZL3b-n.txt')]:
        path = ROOT / filename
        data = path.read_bytes()
        lines = data.decode('utf-8-sig').splitlines()
        rows = []
        for line in lines:
            m = re.match(r'<f1v\.(\d+),([^>]+)>\s+(.*)', line)
            if m:
                rows.append({'locus': int(m[1]), 'code': m[2], 'raw': m[3]})
        assert [r['locus'] for r in rows] == list(range(1, 11))
        assert all(r['raw'].count('<->') == 1 for r in rows)
        starts = [r['locus'] for r in rows if '<%>' in r['raw']]
        assert starts == [1, 5]
        result['sources'][edition] = {
            'url': 'https://voynich.nu/data/' + filename,
            'version_header': lines[:4], 'downloaded': '2026-09-17',
            'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
            'loci': rows, 'paragraph_starts': starts, 'paragraph_lengths': [4, 6],
            'drawing_interruptions': 10,
        }
        for split in [False, True]:
            records = []
            for row in rows:
                left, right = [normalize(part, split) for part in row['raw'].split('<->')]
                broken = normalize(row['raw'], split)
                assert left and right
                assert len(left) + len(right) == len(broken) + 1
                records.append({
                    'locus': row['locus'], 'left_tokens': left, 'right_tokens': right,
                    'before_drawing': left[-1], 'after_drawing': right[0],
                    'token_count': len(left) + len(right),
                    'broken_strip_tags_count': len(broken),
                    'spurious_fused_token': left[-1] + right[0],
                })
            run_id = edition + ('_split' if split else '_join')
            result['runs'][run_id] = {
                'total_tokens': sum(r['token_count'] for r in records),
                'left_tokens': sum(len(r['left_tokens']) for r in records),
                'right_tokens': sum(len(r['right_tokens']) for r in records),
                'broken_strip_tags_total': sum(r['broken_strip_tags_count'] for r in records),
                'tokens_lost_by_bad_gap_handling': 10,
                'rows': records,
            }
    result['regression_checks']['artificial_gap'] = {
        'input': 'ol<->oltchey', 'correct_segments': [normalize('ol', False), normalize('oltchey', False)],
        'wrong_generic_tag_removal': normalize('ol<->oltchey', False),
    }
    baseline = ROOT / 'f1r_v005_source_excerpt.json'
    if baseline.exists():
        old = json.loads(baseline.read_text())
        count = sum(line.count('<->') for edition in ['IT', 'ZL'] for line in old[edition]['records'])
        assert count == 0
        result['regression_checks']['f1r_v005_gap_count'] = count
        result['regression_checks']['f1r_scope'] = 'The new gap-preservation issue does not alter the f1r v005 counts: its stored input has no <-> markers.'
    (ROOT / 'T-SEG-001_results.json').write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
    print(json.dumps({k: {a:b for a,b in v.items() if a != 'rows'} for k,v in result['runs'].items()}, indent=2))

if __name__ == '__main__':
    main()