pstruebi
2845458d3f
- Add .gitignore for Python, virtual environments, testing artifacts, IDE files, and LC3 test outputs including SQAM audio files - Add AGENTS.md with project context for LC3 implementation testing - Add LC3.TS.p5.pdf test specification document - Add LC3 conformance interoperability test software V1.0.8 with script readme and reference binary symlink
294 lines
11 KiB
Python
294 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Run LC3 ENC conformance tests and collect all outputs into a timestamped results folder.
|
|
|
|
Usage (from lc3_quali root):
|
|
poetry run python qualification/run_enc_tests.py
|
|
|
|
Each run creates:
|
|
qualification/results/DD_MM_YY_HH_MM/
|
|
SUMMARY.html ← consultant-ready: TC IDs + PASS/FAIL
|
|
testcases_list.txt ← reference list copy
|
|
ENC_narrow_10ms_<ts>.html ← detail reports per section
|
|
ENC_fb_10ms_<ts>.html
|
|
...
|
|
conformanceCheck_<ts>.log
|
|
lc3_conformance_<ts>/ ← audio comparison files (kept)
|
|
ENC_narrow_10ms/
|
|
encode_ABBA_8000_24000_ref_ref.wav ← reference encoder → reference decoder
|
|
encode_ABBA_8000_24000_tst_ref.wav ← OUR encoder → reference decoder
|
|
...
|
|
"""
|
|
|
|
import datetime
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
QUAL_DIR = pathlib.Path(__file__).resolve().parent
|
|
REPO_ROOT = QUAL_DIR.parent
|
|
CONFORMANCE_DIR = (
|
|
REPO_ROOT
|
|
/ 'LC3_conformance_interoperability_test_software_V1.0.8_2024-07-01'
|
|
/ 'LC3_Conformance_Interoperability_Script'
|
|
)
|
|
RESULTS_BASE = QUAL_DIR / 'results'
|
|
CFGS = [
|
|
'conf_lc3ts_p5_enc_10ms.cfg',
|
|
'conf_lc3ts_p5_enc_75ms.cfg',
|
|
]
|
|
|
|
# Maps cfg section name → {(sample_rate_hz, bitrate_bps): (tc_id, description)}
|
|
# TC IDs as per LC3 TS p5
|
|
# TC_ORDER: band-grouped display order — (band_label, section, fs_hz, br_bps)
|
|
TC_ORDER = [
|
|
('NB', 'ENC_narrow_10ms', 8000, 24000),
|
|
('NB', 'ENC_nb_75ms', 8000, 27734),
|
|
('WB', 'ENC_narrow_10ms', 16000, 32000),
|
|
('WB', 'ENC_narrow_75ms', 16000, 32000),
|
|
('SSWB', 'ENC_narrow_10ms', 24000, 48000),
|
|
('SSWB', 'ENC_narrow_75ms', 24000, 48000),
|
|
('SWB', 'ENC_narrow_10ms', 32000, 64000),
|
|
('SWB', 'ENC_narrow_75ms', 32000, 64000),
|
|
('SWB', 'ENC_hfp_75ms', 32000, 61867),
|
|
('FB', 'ENC_fb_10ms', 48000, 80000),
|
|
('FB', 'ENC_fb_10ms', 48000, 96000),
|
|
('FB', 'ENC_fb_10ms', 48000, 124000),
|
|
('FB', 'ENC_fb_75ms', 48000, 80000),
|
|
('FB', 'ENC_fb_75ms', 48000, 96000),
|
|
('FB', 'ENC_fb_75ms', 48000, 124800),
|
|
]
|
|
|
|
TC_MAP = {
|
|
'ENC_narrow_10ms': {
|
|
(8000, 24000): ('LC3/ENC/NB/BV-01-C', 'NB, 8 kHz, 24 kbps, 10 ms'),
|
|
(16000, 32000): ('LC3/ENC/WB/BV-01-C', 'WB, 16 kHz, 32 kbps, 10 ms'),
|
|
(24000, 48000): ('LC3/ENC/SSWB/BV-01-C', 'SSWB, 24 kHz, 48 kbps, 10 ms'),
|
|
(32000, 64000): ('LC3/ENC/SWB/BV-01-C', 'SWB, 32 kHz, 64 kbps, 10 ms'),
|
|
},
|
|
'ENC_fb_10ms': {
|
|
(48000, 80000): ('LC3/ENC/FB/BV-01-C', 'FB, 48 kHz, 80 kbps, 10 ms'),
|
|
(48000, 96000): ('LC3/ENC/FB/BV-02-C', 'FB, 48 kHz, 96 kbps, 10 ms'),
|
|
(48000, 124000): ('LC3/ENC/FB/BV-03-C', 'FB, 48 kHz, 124 kbps, 10 ms'),
|
|
},
|
|
'ENC_nb_75ms': {
|
|
(8000, 27734): ('LC3/ENC/NB/BV-02-C', 'NB, 8 kHz, 27.734 kbps, 7.5 ms'),
|
|
},
|
|
'ENC_narrow_75ms': {
|
|
(16000, 32000): ('LC3/ENC/WB/BV-02-C', 'WB, 16 kHz, 32 kbps, 7.5 ms'),
|
|
(24000, 48000): ('LC3/ENC/SSWB/BV-02-C', 'SSWB, 24 kHz, 48 kbps, 7.5 ms'),
|
|
(32000, 64000): ('LC3/ENC/SWB/BV-02-C', 'SWB, 32 kHz, 64 kbps, 7.5 ms'),
|
|
},
|
|
'ENC_hfp_75ms': {
|
|
(32000, 61867): ('LC3/ENC/SWB/BV-03-C', 'SWB, 32 kHz, 61.867 kbps, 7.5 ms'),
|
|
},
|
|
'ENC_fb_75ms': {
|
|
(48000, 80000): ('LC3/ENC/FB/BV-07-C', 'FB, 48 kHz, 80 kbps, 7.5 ms'),
|
|
(48000, 96000): ('LC3/ENC/FB/BV-08-C', 'FB, 48 kHz, 96 kbps, 7.5 ms'),
|
|
(48000, 124800): ('LC3/ENC/FB/BV-09-C', 'FB, 48 kHz, 124.8 kbps, 7.5 ms'),
|
|
},
|
|
}
|
|
|
|
SUMMARY_STYLE = """
|
|
body { font-family: sans-serif; max-width: 960px; margin: 2em auto; color: #222; }
|
|
h1 { margin-bottom: 0.2em; }
|
|
.meta { color: #555; font-size: 0.9em; margin-bottom: 1.5em; }
|
|
.banner { font-size: 1.3em; font-weight: bold; padding: 0.5em 1em;
|
|
border-radius: 4px; display: inline-block; margin-bottom: 1.5em; }
|
|
.pass-banner { background: #d4edda; color: #155724; border: 2px solid #28a745; }
|
|
.fail-banner { background: #f8d7da; color: #721c24; border: 2px solid #dc3545; }
|
|
table { border-collapse: collapse; width: 100%; margin-bottom: 2em; }
|
|
th { background: #343a40; color: #fff; padding: 0.6em 0.8em; text-align: left; }
|
|
td { padding: 0.5em 0.8em; border-bottom: 1px solid #dee2e6; font-size: 0.92em; }
|
|
tr:nth-child(even) { background: #f8f9fa; }
|
|
.pass { color: #155724; font-weight: bold; }
|
|
.fail { color: #721c24; font-weight: bold; }
|
|
.notrun { color: #856404; font-weight: bold; }
|
|
code { font-family: monospace; font-size: 0.88em; }
|
|
.note { background: #fff3cd; border: 1px solid #ffc107; border-radius: 4px;
|
|
padding: 0.8em 1em; font-size: 0.88em; margin-top: 1em; }
|
|
.group-header td { background: #e9ecef; font-weight: bold; color: #343a40;
|
|
padding: 0.35em 0.8em; font-size: 0.85em; letter-spacing: 0.05em; }
|
|
"""
|
|
|
|
|
|
def section_from_stem(stem: str) -> str:
|
|
"""Extract section name from HTML file stem.
|
|
|
|
conformanceCheck writes: {section}_{YYYY-MM-DD}_{HH-MM}.html
|
|
e.g. ENC_narrow_10ms_2026-02-27_14-30
|
|
Last two underscore-parts are date and time → strip them.
|
|
"""
|
|
parts = stem.split('_')
|
|
return '_'.join(parts[:-2])
|
|
|
|
|
|
def parse_html_results(html_path: pathlib.Path) -> dict:
|
|
"""Return {(fs_hz, br_bps): 'PASS'|'FAIL'} by parsing the conformance HTML table."""
|
|
content = html_path.read_text(encoding='utf-8', errors='replace')
|
|
|
|
row_re = re.compile(
|
|
r'<tr><td>encode</td><td>[^<]+</td>'
|
|
r'<td>(\d+)</td><td>(\d+)</td>(.*?)</tr>',
|
|
re.DOTALL,
|
|
)
|
|
cell_class_re = re.compile(r'class=(pass|fail)')
|
|
|
|
config_rows: dict = {}
|
|
for m in row_re.finditer(content):
|
|
fs, br = int(m.group(1)), int(m.group(2))
|
|
classes = [c.group(1) for c in cell_class_re.finditer(m.group(3))]
|
|
config_rows.setdefault((fs, br), []).append(
|
|
bool(classes) and all(c == 'pass' for c in classes)
|
|
)
|
|
|
|
return {k: ('PASS' if all(v) else 'FAIL') for k, v in config_rows.items()}
|
|
|
|
|
|
def generate_summary(run_dir: pathlib.Path, html_files: list, run_ts: str) -> bool:
|
|
"""Parse detail HTMLs, build SUMMARY.html. Returns True if all TCs passed."""
|
|
|
|
# Collect per-(section, fs, br) results
|
|
section_results: dict = {}
|
|
for hp in html_files:
|
|
section = section_from_stem(hp.stem)
|
|
if section in TC_MAP:
|
|
section_results[section] = parse_html_results(hp)
|
|
|
|
# Build ordered table rows (band-grouped via TC_ORDER)
|
|
tc_rows = [] # (group_label|None, tc_id, desc, result, detail_html)
|
|
overall_pass = True
|
|
prev_band = None
|
|
for band, section, fs, br in TC_ORDER:
|
|
tc_id, desc = TC_MAP[section][(fs, br)]
|
|
if section in section_results:
|
|
result = section_results[section].get((fs, br), 'NOT RUN')
|
|
else:
|
|
result = 'NOT RUN'
|
|
detail_html = next(
|
|
(h.name for h in html_files if section_from_stem(h.stem) == section),
|
|
'#',
|
|
)
|
|
if result != 'PASS':
|
|
overall_pass = False
|
|
group_header = band if band != prev_band else None
|
|
prev_band = band
|
|
tc_rows.append((group_header, tc_id, desc, result, detail_html))
|
|
|
|
banner_cls = 'pass-banner' if overall_pass else 'fail-banner'
|
|
banner_txt = '✓ ALL 15 TEST CASES PASSED' if overall_pass else '✗ FAILURES DETECTED — see detail reports'
|
|
|
|
rows_html = ''
|
|
for group_header, tc_id, desc, result, detail in tc_rows:
|
|
if group_header:
|
|
rows_html += (
|
|
f'<tr class="group-header">'
|
|
f'<td colspan="4">{group_header}</td>'
|
|
f'</tr>\n'
|
|
)
|
|
res_cls = {'PASS': 'pass', 'FAIL': 'fail'}.get(result, 'notrun')
|
|
rows_html += (
|
|
f'<tr>'
|
|
f'<td><code>{tc_id}</code></td>'
|
|
f'<td>{desc}</td>'
|
|
f'<td class="{res_cls}">{result}</td>'
|
|
f'<td><a href="{detail}">Detail →</a></td>'
|
|
f'</tr>\n'
|
|
)
|
|
|
|
html = f"""<!DOCTYPE html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8">
|
|
<title>LC3 ENC Conformance Summary — {run_ts}</title>
|
|
<style>{SUMMARY_STYLE}</style>
|
|
</head><body>
|
|
<h1>LC3 Encoder Conformance Summary</h1>
|
|
<div class="meta">
|
|
<strong>Run:</strong> {run_ts} |
|
|
<strong>Standard:</strong> LC3 TS p5 |
|
|
<strong>Scope:</strong> ENC — 15 test cases |
|
|
<strong>Pass criterion:</strong> PEAQ ODG Δ ≤ −0.07
|
|
</div>
|
|
<div class="banner {banner_cls}">{banner_txt}</div>
|
|
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Test Case ID (LC3 TS p5)</th>
|
|
<th>Operating Point</th>
|
|
<th>Result</th>
|
|
<th>Detail Report</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{rows_html} </tbody>
|
|
</table>
|
|
|
|
<div class="note">
|
|
<strong>Audio comparison files</strong> are in <code>lc3_conformance_*/<section>/</code>:<br>
|
|
• <code>encode_<ITEM>_<fs>_<br>_ref_ref.wav</code>
|
|
— Reference encoder → Reference decoder (<em>golden reference</em>)<br>
|
|
• <code>encode_<ITEM>_<fs>_<br>_tst_ref.wav</code>
|
|
— <strong>Our encoder</strong> → Reference decoder (<em>device under test output</em>)<br>
|
|
SQAM test items used: ABBA, Castanets, Eddie_Rabbitt, Female_Speech_German,
|
|
Glockenspiel, Piano_Schubert, Violoncello, Harpsichord, Male_Speech_English.
|
|
</div>
|
|
</body></html>"""
|
|
|
|
(run_dir / 'SUMMARY.html').write_text(html, encoding='utf-8')
|
|
return overall_pass
|
|
|
|
|
|
def collect_outputs(run_dir: pathlib.Path) -> list:
|
|
"""Move HTML/log/work-dir files from conformance dir into run_dir. Returns list of HTML paths."""
|
|
html_files = []
|
|
for item in list(CONFORMANCE_DIR.iterdir()):
|
|
if item.suffix == '.html' or item.suffix == '.log' or item.name.startswith('lc3_conformance_'):
|
|
dest = run_dir / item.name
|
|
shutil.move(str(item), str(dest))
|
|
if item.suffix == '.html':
|
|
html_files.append(dest)
|
|
return html_files
|
|
|
|
|
|
def main() -> None:
|
|
run_ts = datetime.datetime.now().strftime('%d_%m_%y_%H_%M')
|
|
run_dir = RESULTS_BASE / f'lc3_quali_{run_ts}'
|
|
run_dir.mkdir(parents=True, exist_ok=True)
|
|
print(f'Results folder: {run_dir}')
|
|
|
|
wine_env = os.environ.copy()
|
|
wine_env['WINEARCH'] = 'win32'
|
|
wine_env['WINEPREFIX'] = str(pathlib.Path.home() / '.wine32')
|
|
|
|
subprocess_ok = True
|
|
for cfg in CFGS:
|
|
print(f'\n{"=" * 60}')
|
|
print(f'Running: {cfg}')
|
|
print('=' * 60)
|
|
r = subprocess.run(
|
|
[sys.executable, 'conformanceCheck.py', cfg, '-keep', '-system_sox'],
|
|
cwd=CONFORMANCE_DIR,
|
|
env=wine_env,
|
|
)
|
|
if r.returncode != 0:
|
|
print(f'ERROR: conformanceCheck.py exited with code {r.returncode}', file=sys.stderr)
|
|
subprocess_ok = False
|
|
|
|
html_files = collect_outputs(run_dir)
|
|
shutil.copy(str(QUAL_DIR / 'testcases_list.txt'), str(run_dir / 'testcases_list.txt'))
|
|
|
|
overall_pass = generate_summary(run_dir, html_files, run_ts)
|
|
|
|
print(f'\n{"=" * 60}')
|
|
print(f'Subprocess OK : {subprocess_ok}')
|
|
print(f'Conformance : {"ALL PASSED" if overall_pass else "FAILURES — check SUMMARY.html"}')
|
|
print(f'Results folder: {run_dir}')
|
|
print('=' * 60)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|