@article{2026discAI,
 abstract = {Large language models (LLMs) are increasingly considered for scientific evaluation workflows, including editorial triage, screening, and ranking. This raises a fairness and reliability question: are LLM-based evaluators counterfactually invariant to extrinsic author-context metadata when manuscript content is held fixed? We conduct a controlled counterfactual audit of 80 recent English-language arXiv manuscripts from computer science, mathematics, physics, and quantitative biology. For each manuscript, we append a corresponding-author profile in a {\$}{\$}2{\backslash}times 2{\backslash}times 2{\$}{\$}factorial design varying name-coded identity, institution prestige tier, and bibliometric profile, alongside a no-metadata BLIND baseline. Six heterogeneous LLM evaluators assign ratings on 14 review-style criteria and three summary outcomes: content quality, perceived prestige/impact, and acceptance recommendation. Estimates are computed at the manuscript level after averaging repeated calls within model and pooling evaluator models with equal weight; uncertainty is assessed with bootstrap confidence intervals, sign-flip tests, false-discovery-rate control, and model heterogeneity analyses. Evaluator outputs are not invariant to author metadata. Bibliometric and institution-tier cues systematically increase perceived prestige/impact and acceptance-related judgments, while effects on content-quality judgments and name-coded effects are smaller. Ranking analyses show that metadata perturbations can displace manuscripts and alter top-K shortlist membership. These results identify a concrete reliability risk for LLM-assisted scientific evaluation pipelines.},
 article-number = {1404},
 author = {Marco Rospocher},
 bdsk-url-1 = {https://doi.org/10.1007/s44163-026-02268-y},
 date-added = {2026-10-08 13:17:59 +0200},
 date-modified = {2026-10-16 09:57:44 +0200},
 day = {08},
 doi = {10.1007/s44163-026-02268-y},
 issn = {2731-0809},
 journal = {Discover Artificial Intelligence},
 month = {Oct},
 number = {1},
 publisher = {Springer Nature},
 title = {Author metadata affects Large Language Model scores in scientific peer review},
 volume = {6},
 year = {2026}
}