#!/usr/bin/env python3 """Run LC3 ENC conformance tests and collect all outputs into a timestamped results folder. Usage (from lc3_quali root): poetry run python qualification/run_enc_tests.py Each run creates: qualification/results/DD_MM_YY_HH_MM/ SUMMARY.html ← consultant-ready: TC IDs + PASS/FAIL testcases_list.txt ← reference list copy ENC_narrow_10ms_.html ← detail reports per section ENC_fb_10ms_.html ... conformanceCheck_.log lc3_conformance_/ ← audio comparison files (kept) ENC_narrow_10ms/ encode_ABBA_8000_24000_ref_ref.wav ← reference encoder → reference decoder encode_ABBA_8000_24000_tst_ref.wav ← OUR encoder → reference decoder ... """ import datetime import os import pathlib import re import shutil import subprocess import sys QUAL_DIR = pathlib.Path(__file__).resolve().parent REPO_ROOT = QUAL_DIR.parent CONFORMANCE_DIR = ( REPO_ROOT / 'LC3_conformance_interoperability_test_software_V1.0.8_2024-07-01' / 'LC3_Conformance_Interoperability_Script' ) RESULTS_BASE = QUAL_DIR / 'results' CFGS = [ 'conf_lc3ts_p5_enc_10ms.cfg', 'conf_lc3ts_p5_enc_75ms.cfg', ] # Maps cfg section name → {(sample_rate_hz, bitrate_bps): (tc_id, description)} # TC IDs as per LC3 TS p5 # TC_ORDER: band-grouped display order — (band_label, section, fs_hz, br_bps) TC_ORDER = [ ('NB', 'ENC_narrow_10ms', 8000, 24000), ('NB', 'ENC_nb_75ms', 8000, 27734), ('WB', 'ENC_narrow_10ms', 16000, 32000), ('WB', 'ENC_narrow_75ms', 16000, 32000), ('SSWB', 'ENC_narrow_10ms', 24000, 48000), ('SSWB', 'ENC_narrow_75ms', 24000, 48000), ('SWB', 'ENC_narrow_10ms', 32000, 64000), ('SWB', 'ENC_narrow_75ms', 32000, 64000), ('SWB', 'ENC_hfp_75ms', 32000, 61867), ('FB', 'ENC_fb_10ms', 48000, 80000), ('FB', 'ENC_fb_10ms', 48000, 96000), ('FB', 'ENC_fb_10ms', 48000, 124000), ('FB', 'ENC_fb_75ms', 48000, 80000), ('FB', 'ENC_fb_75ms', 48000, 96000), ('FB', 'ENC_fb_75ms', 48000, 124800), ] TC_MAP = { 'ENC_narrow_10ms': { (8000, 24000): ('LC3/ENC/NB/BV-01-C', 'NB, 8 kHz, 24 kbps, 10 ms'), (16000, 32000): ('LC3/ENC/WB/BV-01-C', 'WB, 16 kHz, 32 kbps, 10 ms'), (24000, 48000): ('LC3/ENC/SSWB/BV-01-C', 'SSWB, 24 kHz, 48 kbps, 10 ms'), (32000, 64000): ('LC3/ENC/SWB/BV-01-C', 'SWB, 32 kHz, 64 kbps, 10 ms'), }, 'ENC_fb_10ms': { (48000, 80000): ('LC3/ENC/FB/BV-01-C', 'FB, 48 kHz, 80 kbps, 10 ms'), (48000, 96000): ('LC3/ENC/FB/BV-02-C', 'FB, 48 kHz, 96 kbps, 10 ms'), (48000, 124000): ('LC3/ENC/FB/BV-03-C', 'FB, 48 kHz, 124 kbps, 10 ms'), }, 'ENC_nb_75ms': { (8000, 27734): ('LC3/ENC/NB/BV-02-C', 'NB, 8 kHz, 27.734 kbps, 7.5 ms'), }, 'ENC_narrow_75ms': { (16000, 32000): ('LC3/ENC/WB/BV-02-C', 'WB, 16 kHz, 32 kbps, 7.5 ms'), (24000, 48000): ('LC3/ENC/SSWB/BV-02-C', 'SSWB, 24 kHz, 48 kbps, 7.5 ms'), (32000, 64000): ('LC3/ENC/SWB/BV-02-C', 'SWB, 32 kHz, 64 kbps, 7.5 ms'), }, 'ENC_hfp_75ms': { (32000, 61867): ('LC3/ENC/SWB/BV-03-C', 'SWB, 32 kHz, 61.867 kbps, 7.5 ms'), }, 'ENC_fb_75ms': { (48000, 80000): ('LC3/ENC/FB/BV-07-C', 'FB, 48 kHz, 80 kbps, 7.5 ms'), (48000, 96000): ('LC3/ENC/FB/BV-08-C', 'FB, 48 kHz, 96 kbps, 7.5 ms'), (48000, 124800): ('LC3/ENC/FB/BV-09-C', 'FB, 48 kHz, 124.8 kbps, 7.5 ms'), }, } SUMMARY_STYLE = """ body { font-family: sans-serif; max-width: 960px; margin: 2em auto; color: #222; } h1 { margin-bottom: 0.2em; } .meta { color: #555; font-size: 0.9em; margin-bottom: 1.5em; } .banner { font-size: 1.3em; font-weight: bold; padding: 0.5em 1em; border-radius: 4px; display: inline-block; margin-bottom: 1.5em; } .pass-banner { background: #d4edda; color: #155724; border: 2px solid #28a745; } .fail-banner { background: #f8d7da; color: #721c24; border: 2px solid #dc3545; } table { border-collapse: collapse; width: 100%; margin-bottom: 2em; } th { background: #343a40; color: #fff; padding: 0.6em 0.8em; text-align: left; } td { padding: 0.5em 0.8em; border-bottom: 1px solid #dee2e6; font-size: 0.92em; } tr:nth-child(even) { background: #f8f9fa; } .pass { color: #155724; font-weight: bold; } .fail { color: #721c24; font-weight: bold; } .notrun { color: #856404; font-weight: bold; } code { font-family: monospace; font-size: 0.88em; } .note { background: #fff3cd; border: 1px solid #ffc107; border-radius: 4px; padding: 0.8em 1em; font-size: 0.88em; margin-top: 1em; } .group-header td { background: #e9ecef; font-weight: bold; color: #343a40; padding: 0.35em 0.8em; font-size: 0.85em; letter-spacing: 0.05em; } """ def section_from_stem(stem: str) -> str: """Extract section name from HTML file stem. conformanceCheck writes: {section}_{YYYY-MM-DD}_{HH-MM}.html e.g. ENC_narrow_10ms_2026-02-27_14-30 Last two underscore-parts are date and time → strip them. """ parts = stem.split('_') return '_'.join(parts[:-2]) def parse_html_results(html_path: pathlib.Path) -> dict: """Return {(fs_hz, br_bps): 'PASS'|'FAIL'} by parsing the conformance HTML table.""" content = html_path.read_text(encoding='utf-8', errors='replace') row_re = re.compile( r'encode[^<]+' r'(\d+)(\d+)(.*?)', re.DOTALL, ) cell_class_re = re.compile(r'class=(pass|fail)') config_rows: dict = {} for m in row_re.finditer(content): fs, br = int(m.group(1)), int(m.group(2)) classes = [c.group(1) for c in cell_class_re.finditer(m.group(3))] config_rows.setdefault((fs, br), []).append( bool(classes) and all(c == 'pass' for c in classes) ) return {k: ('PASS' if all(v) else 'FAIL') for k, v in config_rows.items()} def generate_summary(run_dir: pathlib.Path, html_files: list, run_ts: str) -> bool: """Parse detail HTMLs, build SUMMARY.html. Returns True if all TCs passed.""" # Collect per-(section, fs, br) results section_results: dict = {} for hp in html_files: section = section_from_stem(hp.stem) if section in TC_MAP: section_results[section] = parse_html_results(hp) # Build ordered table rows (band-grouped via TC_ORDER) tc_rows = [] # (group_label|None, tc_id, desc, result, detail_html) overall_pass = True prev_band = None for band, section, fs, br in TC_ORDER: tc_id, desc = TC_MAP[section][(fs, br)] if section in section_results: result = section_results[section].get((fs, br), 'NOT RUN') else: result = 'NOT RUN' detail_html = next( (h.name for h in html_files if section_from_stem(h.stem) == section), '#', ) if result != 'PASS': overall_pass = False group_header = band if band != prev_band else None prev_band = band tc_rows.append((group_header, tc_id, desc, result, detail_html)) banner_cls = 'pass-banner' if overall_pass else 'fail-banner' banner_txt = '✓ ALL 15 TEST CASES PASSED' if overall_pass else '✗ FAILURES DETECTED — see detail reports' rows_html = '' for group_header, tc_id, desc, result, detail in tc_rows: if group_header: rows_html += ( f'' f'{group_header}' f'\n' ) res_cls = {'PASS': 'pass', 'FAIL': 'fail'}.get(result, 'notrun') rows_html += ( f'' f'{tc_id}' f'{desc}' f'{result}' f'Detail →' f'\n' ) html = f""" LC3 ENC Conformance Summary — {run_ts}

LC3 Encoder Conformance Summary

Run: {run_ts}  |  Standard: LC3 TS p5  |  Scope: ENC — 15 test cases  |  Pass criterion: PEAQ ODG Δ ≤ −0.07
{rows_html}
Test Case ID (LC3 TS p5) Operating Point Result Detail Report
Audio comparison files are in lc3_conformance_*/<section>/:
encode_<ITEM>_<fs>_<br>_ref_ref.wav — Reference encoder → Reference decoder  (golden reference)
encode_<ITEM>_<fs>_<br>_tst_ref.wavOur encoder → Reference decoder  (device under test output)
SQAM test items used: ABBA, Castanets, Eddie_Rabbitt, Female_Speech_German, Glockenspiel, Piano_Schubert, Violoncello, Harpsichord, Male_Speech_English.
""" (run_dir / 'SUMMARY.html').write_text(html, encoding='utf-8') return overall_pass def collect_outputs(run_dir: pathlib.Path) -> list: """Move HTML/log/work-dir files from conformance dir into run_dir. Returns list of HTML paths.""" html_files = [] for item in list(CONFORMANCE_DIR.iterdir()): if item.suffix == '.html' or item.suffix == '.log' or item.name.startswith('lc3_conformance_'): dest = run_dir / item.name shutil.move(str(item), str(dest)) if item.suffix == '.html': html_files.append(dest) return html_files def main() -> None: run_ts = datetime.datetime.now().strftime('%d_%m_%y_%H_%M') run_dir = RESULTS_BASE / f'lc3_quali_{run_ts}' run_dir.mkdir(parents=True, exist_ok=True) print(f'Results folder: {run_dir}') wine_env = os.environ.copy() wine_env['WINEARCH'] = 'win32' wine_env['WINEPREFIX'] = str(pathlib.Path.home() / '.wine32') subprocess_ok = True for cfg in CFGS: print(f'\n{"=" * 60}') print(f'Running: {cfg}') print('=' * 60) r = subprocess.run( [sys.executable, 'conformanceCheck.py', cfg, '-keep', '-system_sox'], cwd=CONFORMANCE_DIR, env=wine_env, ) if r.returncode != 0: print(f'ERROR: conformanceCheck.py exited with code {r.returncode}', file=sys.stderr) subprocess_ok = False html_files = collect_outputs(run_dir) shutil.copy(str(QUAL_DIR / 'testcases_list.txt'), str(run_dir / 'testcases_list.txt')) overall_pass = generate_summary(run_dir, html_files, run_ts) print(f'\n{"=" * 60}') print(f'Subprocess OK : {subprocess_ok}') print(f'Conformance : {"ALL PASSED" if overall_pass else "FAILURES — check SUMMARY.html"}') print(f'Results folder: {run_dir}') print('=' * 60) if __name__ == '__main__': main()