{
  "title": "Beyond Character Error Rate: Evaluating Diacritic Fidelity and Post-Processing Effects in Classical Arabic OCR",
  "author": {
    "name": "Mohammed Fazalullah Qudrath",
    "affiliation": "Independent Researcher",
    "orcid": "https://orcid.org/0009-0003-3684-3130"
  },
  "status": "preprint (no arXiv identifier or DOI assigned at export time)",
  "export_date": "2026-10-26",
  "systems": {
    "base": {
      "model": "dots-studio/dots.mocr",
      "url": "https://huggingface.co/dots-studio/dots.mocr",
      "revision": "not pinned in code; e539fbb inferred from hub history"
    },
    "v0.1": {
      "model": "pandanlabs/dots-mocr-arabic-classical",
      "url": "https://huggingface.co/pandanlabs/dots-mocr-arabic-classical",
      "revision": "037b472f2a2fd004e02ce012549844af119eea62",
      "training_run": "lora-001",
      "identity_with_evaluated_checkpoint": "inferred from storage path, timestamps and shard sizes; not hash-verified"
    },
    "v0.1 + retry": "same weights; a page that hits the 4,096-token limit is re-sent once with repetition penalty 1.1",
    "lora-002": "second training run; not released"
  },
  "training_shared": {
    "method": "LoRA on the language model only; vision encoder frozen",
    "rank": 16,
    "alpha": 32,
    "dropout": 0.05,
    "optimizer": "AdamW",
    "learning_rate": 0.0001,
    "schedule": "cosine with warmup",
    "batch_size": 1,
    "gradient_accumulation": 8,
    "precision": "bf16",
    "image_long_side_px": 2048,
    "checkpoint_selection": "lowest validation loss on the 36 validation pages",
    "seed": "not logged (code default 1)"
  },
  "inference": {
    "server": "vLLM 0.30",
    "decoding": "greedy (temperature 0)",
    "max_new_tokens": 4096,
    "prompt": "Extract the text content from this image."
  },
  "evaluation_sets": {
    "kashf": {
      "label": "Kashf ash-Shubuhat (typeset)",
      "role": "test",
      "reference_status": "human-verified",
      "condition": "typeset",
      "pages_scored_cer": "80",
      "pages_paired_der": "80"
    },
    "taghut": {
      "label": "Sharh Ma'na at-Taghut (low-resolution scan)",
      "role": "test",
      "reference_status": "human-verified",
      "condition": "scan, low resolution",
      "pages_scored_cer": "44",
      "pages_paired_der": "42"
    },
    "goldtest": {
      "label": "Gold test split, all",
      "role": "test",
      "reference_status": "human-verified",
      "condition": "typeset + low-resolution scan",
      "pages_scored_cer": "124",
      "pages_paired_der": "122"
    },
    "unvver": {
      "label": "Unvowelled scans, verified subset",
      "role": "additional evaluation",
      "reference_status": "human-verified",
      "condition": "scans",
      "pages_scored_cer": "38",
      "pages_paired_der": "35"
    },
    "hires": {
      "label": "All 300 dpi pages (59 = 10-page sample + 49 Inna Kunna pages)",
      "role": "additional evaluation",
      "reference_status": "silver (unverified)",
      "condition": "scan, 300 dpi",
      "pages_scored_cer": "59",
      "pages_paired_der": "58"
    },
    "jami": {
      "label": "Sharh al-Jami' (low-resolution scan)",
      "role": "additional evaluation",
      "reference_status": "silver (unverified)",
      "condition": "scan, low resolution",
      "pages_scored_cer": "48",
      "pages_paired_der": "45"
    },
    "validation": {
      "label": "Validation (checkpoint selection; not independent)",
      "role": "selection",
      "reference_status": "human-verified",
      "condition": "typeset",
      "pages_scored_cer": "36",
      "pages_paired_der": "36"
    },
    "hfont": {
      "label": "Synthetic renders, held-out font",
      "role": "synthetic",
      "reference_status": "synthetic (exact)",
      "condition": "rendered",
      "pages_scored_cer": "60",
      "pages_paired_der": "59"
    },
    "htext": {
      "label": "Synthetic renders, held-out text",
      "role": "synthetic",
      "reference_status": "synthetic (exact)",
      "condition": "rendered",
      "pages_scored_cer": "60",
      "pages_paired_der": "59"
    }
  },
  "metrics": {
    "cer_letters": "Levenshtein distance / reference length on the letters view (diacritics, Qur'anic annotation marks and tatweel removed; NFC; no letter folding); page mean",
    "der": "share of aligned Arabic letters, marked on at least one side, whose sorted harakat differ; letters the alignment does not match are excluded; undefined when no aligned letter is marked on either side; page mean over defined pages",
    "invented_marks_per_1k": "aligned letters bare in the reference and marked in the output, per 1,000 aligned letters, pooled",
    "omitted_marks_per_1k": "aligned letters marked in the reference and bare in the output, per 1,000 aligned letters, pooled",
    "bootstrap": "paired, resampling pages; 4,000 resamples; seed 1; percentile 95% interval; describes variation across the evaluated pages only",
    "quote_layer": "deterministic replacement of recognised Qur'anic spans with QPC Hafs text (match score >= 90); the reference is unchanged"
  },
  "bootstrap_seed_sensitivity": [
    {
      "comparison": "base -> lora-001 [quote layer ON]",
      "n": "42",
      "pass_share": "200/200",
      "lower_bound_min": "0.0406",
      "lower_bound_max": "0.0463",
      "notes": "CI lower-bound range over seeds 1..200 (repo evals.Paired, 4000 resamples); tolerance LowResTol=0.05; repo uses seed 1 (lower bound 0.0448)"
    },
    {
      "comparison": "base -> lora-002 [quote layer ON]",
      "n": "41",
      "pass_share": "125/200",
      "lower_bound_min": "0.0472",
      "lower_bound_max": "0.0516",
      "notes": "seed-1 verdict FAIL reproduces exactly; with other seeds the same answers PASS the low-res rule in 125 of 200 seeds"
    }
  ],
  "provenance": {
    "source": "aggregates copied from the audit's recomputed results (paper/evidence/results_recomputed.csv) in the author's private research repository",
    "research_repository": "private (not publicly available)",
    "evaluation_code_commit": "6ce3642 (private repository)",
    "audit_commits": [
      "b348698",
      "064e856"
    ],
    "manuscript_commit_at_export": "see git log of the research branch research/arabic-ocr-paper (private)"
  },
  "public_artifacts": [
    "manuscript.pdf (web edition; scan crops omitted)",
    "figures/figure-1..4.svg",
    "evidence/*.csv",
    "evidence/experiment-manifest.json",
    "citation.bib",
    "https://huggingface.co/pandanlabs/dots-mocr-arabic-classical"
  ],
  "not_public": [
    "training and evaluation code, scorer and quotation layer",
    "page-level predictions and references",
    "page images (rights not cleared)",
    "audit scripts and logs",
    "raw synthetic training renders (deleted; settings reconstructed by replay)"
  ],
  "reproducibility_gaps": [
    "base revision not pinned",
    "training/synthesis command lines and seeds not logged",
    "released weights not hash-verified against the evaluated checkpoint",
    "Taghut references partly unadjudicated",
    "run-to-run inference variability (28/157 answers differ)"
  ]
}