{
  "title": "Gloss-free Auslan translation: evidence snapshot",
  "updated": "2026-09-30",
  "author": "Preet Prasad",
  "dataset": "Auslan-Daily conversational split",
  "metric": "BLEU-4 (not an accuracy percentage)",
  "test_clips": 800,
  "seen_english_target_clips": 230,
  "unseen_english_target_clips": 570,
  "split_source": "phase2_baseline/tools/reproduce_headline.py",
  "systems": [
    {"name": "DWPose + CTC", "official": 9.522, "unseen_target": 4.88, "decoding": "beam 5, length penalty 0.6", "source": "results/semantic_grading/hyps_dwpose_ctc.jsonl"},
    {"name": "VLP-40 (pose-text pretraining, 40 epochs)", "official": 10.272, "unseen_target": 5.31, "decoding": "beam 5, length penalty 0.6", "source": "results/semantic_grading/hyps_vlp.jsonl"},
    {"name": "RGB fusion", "official": 10.335, "unseen_target": 5.81, "decoding": "beam 5, length penalty 0.6", "source": "results/semantic_grading/hyps_rgbfusion.jsonl"},
    {"name": "S6 ensemble", "official": 11.802, "unseen_target": 6.60, "seen_target": 35.00, "official_wer": 0.907, "decoding": "beam 10, length penalty 0.3 chosen on validation", "live": true, "source": "results/dumps_S6_test/hyps_LP=0.3.jsonl"}
  ],
  "ensemble_vs_rgb": {"delta": 1.467, "paired_bootstrap_95_ci": [0.396, 2.563], "p_better": 0.994, "resamples": 2000, "seed": 0, "tool": "phase2_baseline/tools/paired_bootstrap.py", "source": "docs/CLAIMS.md", "scope": "test-sentence sampling only"},
  "reproduction": {"directory": "phase2_baseline", "command": "python tools/reproduce_headline.py", "bootstrap": "docs/REPRODUCIBILITY.md section 0.3", "protocol": "docs/PROTOCOL.md", "report": "docs/REPORT.md", "live_demo_record": "results/live_demo_2026-09-24/README.md"},
  "live_system": {
    "contract": "Utterance-final translation: sign one short utterance, select Finish, then review the English. Recording ends at 10 s or at the frame cap, whichever comes first. The user chooses one of two operating points for each utterance.",
    "capture": "Browser frames carry their capture time and are resampled after Finish to the 25 fps grid of the training clips. DWPose (YOLOX-m detector and RTMW-DW-x-l, 133 points) runs on every frame and follows one signer with the research tracker. On the public site all of this runs in the visitor's browser (a Web Worker running ONNX Runtime Web); run locally, the Python service (demo/service.py) does the same work.",
    "confidence": "Self-CE in nats (lower is more confident). A model answers only when self-CE is at or below its own threshold, fitted on the 800 official validation clips at the chosen answer rate. Thresholds do not transfer between models.",
    "default_answer_rate": 0.5,
    "allowed_answer_rates": [0.5, 0.75, 0.9],
    "operating_points": {
      "single": {
        "label": "Single best model",
        "recipe": "Adopted recipe: real clips plus timing-matched synthetic sequences at 1.1:1 (duration stitch), pose augmentation, deterministic training; wave-4 control arm A, seed 202",
        "selection": "Highest validation BLEU-4 of the 21 adopted-recipe single models scored in job 21736 (7 independent batches x 3 seeds). Its test score is not the highest of the 21 (that is 12.229); selection is on validation.",
        "selection_source": "results/seedens_21736/analysis.txt",
        "input": "pose only",
        "checkpoint": "results/ckpts_live/w4A_s202_best.pt",
        "checkpoint_origin": "Kaya wave4/runs_tq/w4A_s202/best.pt",
        "checkpoint_sha256": "0c245b3701d0c80ce0c8ad79a1c81c964afced5f0ac8c3eabe98f87bd35d65b6",
        "decoding": "beam 10, length penalty 0.7 chosen on validation",
        "validation_bleu4": 12.643,
        "validation_source": "results/live_demo_2026-09-24/dump_w4A_s202_val/hyps_LP=0.7.jsonl",
        "official_test_bleu4": 11.652,
        "official_test_wer": 0.9295,
        "seen_target_test_bleu4": 31.99,
        "unseen_target_test_bleu4": 6.93,
        "test_source": "results/live_demo_2026-09-24/dump_w4A_s202_test/hyps_LP=0.7.jsonl",
        "confidence_threshold_nats": {"0.5": 0.706661, "0.75": 0.859783, "0.9": 0.974299},
        "threshold_source": "results/live_demo_2026-09-24/dump_w4A_s202_val/nbest.jsonl",
        "test_readout_at_validation_threshold": {
          "0.5": {"answered": 0.499, "answered_bleu4": 22.01, "source": "results/live_demo_2026-09-24/dump_w4A_s202_test/nbest.jsonl"},
          "0.75": {"answered": 0.760, "answered_bleu4": 15.27, "source": "results/live_demo_2026-09-24/dump_w4A_s202_test/nbest.jsonl"},
          "0.9": {"answered": 0.915, "answered_bleu4": 12.69, "source": "results/live_demo_2026-09-24/dump_w4A_s202_test/nbest.jsonl"}
        },
        "live_path_check": {"clips": 8, "identical_to_stored_pose_decode": 2, "source": "results/live_demo_2026-09-24/README.md", "note": "Observation on 8 stored validation signer videos run through the live service; an earlier single model (pose augmentation, seed 42) matched on 4 of 8. Not a measure of accuracy."}
      },
      "ensemble": {
        "label": "Six-model ensemble (S6)",
        "members": [
          {"name": "VLP-80", "input": "pose", "checkpoint": "results/ckpts/comm_dwpose_ctc_vlplong80.pt", "sha256": "20a6c5e3119417066a4638be324a77b870e36510aae29ca58b3a279bbfc15b09"},
          {"name": "RGB fusion", "input": "pose + RGB", "checkpoint": "results/ckpts/comm_dwpose_rgb_fusion.pt", "sha256": "a3457758a8c9e645e4b4012d5ebbc2171cb2d1c0a2372fcc21078c87ec83fa8d"},
          {"name": "VLP-40", "input": "pose", "checkpoint": "results/ckpts/comm_dwpose_vlp_ctc.pt", "sha256": "4ca55b0a77868f4146b8c243f428d348d9afe8d266e301c246fc355eb17f23c1"},
          {"name": "VLP → CTC, seed 202 (pose–text pretraining)", "input": "pose", "checkpoint": "results/ckpts/comm_dwpose_ctc_s202.pt", "sha256": "5e7dec289e5b2b7f062ca0989361979be8b0bf46bacb43f8dfa07b7e4dedd62d"},
          {"name": "RGB fusion seed 101", "input": "pose + RGB", "checkpoint": "results/ckpts/comm_dwpose_fus_s101.pt", "sha256": "bed3caeb2a991a6e9dae320f2fb8eaf0319b6470c21b9dc8c3870468393b30e4"},
          {"name": "RGB fusion seed 202", "input": "pose + RGB", "checkpoint": "results/ckpts/comm_dwpose_fus_s202.pt", "sha256": "6582e4ffedee2796e0b8dc056b9d88cd77229df95f0294f708801ef46b8da2fb"}
        ],
        "image_features": "DINOv2-small (revision ed25f3a31f01632728cabb09d1542f84ab7b0056) crops of the signer, left hand and right hand at up to 24 frames, computed after Finish: in the visitor's browser on the public site (the public ONNX export onnx-community/dinov2-small at revision 8b1f705a3a7f6f062f6bdd21986c1583d3ef105d, whose CLS output matches this revision's pooler output to 8.1e-5 on 313 crops), or by the Python service when run locally",
        "decoding": "beam 10, length penalty 0.3 chosen on validation",
        "validation_bleu4": 12.470,
        "validation_source": "results/dumps_S6_val/hyps_LP=0.3.jsonl",
        "official_test_bleu4": 11.802,
        "official_test_wer": 0.907,
        "seen_target_test_bleu4": 35.00,
        "unseen_target_test_bleu4": 6.60,
        "test_source": "results/dumps_S6_test/hyps_LP=0.3.jsonl",
        "confidence_threshold_nats": {"0.5": 0.904779, "0.75": 1.088921, "0.9": 1.236622},
        "threshold_source": "results/dumps_S6_val/nbest.jsonl",
        "test_readout_at_validation_threshold": {
          "0.5": {"answered": 0.519, "answered_bleu4": 19.07, "source": "results/enssel_S6.json"},
          "0.75": {"answered": 0.756, "answered_bleu4": 14.61, "source": "results/enssel_S6.json"},
          "0.9": {"answered": 0.894, "answered_bleu4": 12.90, "source": "results/enssel_S6.json"}
        },
        "live_feature_check": {
          "cosine_vs_stored_800_validation_clips": {"mean_per_crop_slot": {"signer": 0.9994, "left_hand": 0.9971, "right_hand": 0.9970}, "percentile_1": {"signer": 0.9968, "left_hand": 0.9874, "right_hand": 0.9872}, "minimum": {"signer": 0.971, "left_hand": 0.931, "right_hand": 0.876}, "clips_with_a_crop_below_0.9827": "105 of 800", "empty_crops": "476, empty in both"},
          "cosine_vs_stored_first_5_validation_clips": {"min_slot_mean": 0.994, "min_crop": 0.9827, "test": "demo/tests/test_research_parity.py"},
          "validation_bleu4_live_features": 12.291,
          "validation_bleu4_stored_features": 12.470,
          "validation_wer_live_features": 0.9157,
          "validation_wer_stored_features": 0.9164,
          "paired_bootstrap_live_minus_stored": {"delta": -0.179, "ci95": [-0.654, 0.101], "resamples": 2000, "seed": 0, "tool": "phase2_baseline/tools/paired_bootstrap.py"},
          "identical_validation_hypotheses": "772 of 800",
          "source": "results/live_demo_2026-09-24/dump_S6_val_live_rgb/hyps_LP=0.3.jsonl",
          "record": "results/live_demo_2026-09-24/README.md",
          "note": "Live features are close to, not bit-identical with, the stored Kaya features (different library versions). Measured through the Python service; the browser engine's port (demo/static/engine/rgb.js, tested in Node against demo/rgb_features.py on 313 crops: identical crop pixels, features within 8.1e-5) computes the same features, so the browser ensemble inherits this gap."
        },
        "live_path_check": {"clips": 8, "identical_to_stored_pose_decode": 8, "source": "results/live_demo_2026-09-24/README.md", "note": "Observation on 8 stored validation signer videos run through the live service, with live DINOv2 features."}
      }
    },
    "latency_m3max_cpu": {
      "setting": "Python service (demo/service.py) default: 2 torch threads (DEMO_THREADS), ONNX Runtime default threading for pose, uncontended M3 Max CPU",
      "pose_ms_per_frame": "about 170-220",
      "pose_frames_per_second": "about 5",
      "wait_after_finish": "about 4-5 s per second signed",
      "single_translation_ms": "about 30-110",
      "ensemble_decode_s": "about 0.2-0.6, median about 0.29",
      "ensemble_image_features_s": "about 1.0-1.5 for 24 frames",
      "source": "results/live_demo_2026-09-24/README.md",
      "full_pipeline_bench": {"pose_ms_per_frame": 191.9, "pose_frames_per_second": 5.2, "source": "results/pose_throughput_m3max_2026-09-15.json"}
    },
    "speech": {
      "engine": "browser speechSynthesis, local voices only",
      "evaluated_voice": "macOS say -v Karen (en-AU), rate 180, on VLP-40 output",
      "round_trip_wer": {"strict": 0.028, "normalised": 0.011, "source": "results/SPEECH_ROUNDTRIP_2026-09-22.md"},
      "synthesis_ms_median": 660.8,
      "synthesis_source": "results/speech_pipeline_bench.json"
    },
    "nonsigner_check": {"model": "VLP-40 on its own (one of S6's six members, not a standalone live operating point)", "nonsigners_answered": 0.54, "signers_answered": 0.535, "reading": "Confidence does not detect whether anyone is signing.", "source": "results/NONSIGNER_GATE_2026-09-22.md"},
    "not_supported": ["streaming or partial translation", "automatic end-of-utterance detection", "detection of non-signing", "validated accuracy on new webcam signers"]
  },
  "caveats": [
    "Offline benchmark results do not establish webcam performance.",
    "The live interface offers two operating points chosen per utterance: the single best model (pose only) and the six-model ensemble.",
    "The single best model is best on validation among 21 adopted-recipe models; its test score is not the highest of the 21.",
    "Scores at a confidence threshold cover only the answered clips and are not comparable to full-coverage scores.",
    "Answered output skews toward short, formulaic, frequent phrases.",
    "The confidence check does not detect the absence of signing; this was measured on VLP-40 alone, not on the live operating points.",
    "Live ensemble image features are recomputed from the camera frames after Finish (in the visitor's browser on the public site) and agree closely but not exactly with the stored features.",
    "Unseen targets are not necessarily unseen signers; signers other than the two recurring performers scored about 3 BLEU-4 even with the ensemble (docs/CLAIMS.md).",
    "The full source repository may require access; datasets and checkpoints are not redistributed."
  ]
}