Article Instance

HTTP 200 OK
Allow: GET
Content-Type: application/json
Vary: Accept

{
    "pk": 49671,
    "title": "Estimating and Correcting Yes-No Bias in Language Models",
    "subtitle": null,
    "abstract": "When presented with a yes-no question, humans tend to say 'yes' regardless of the ground truth. This 'yes-bias' can be attributed either to the social pressure to agree with an interlocutor or simply to the tendency to mimic the distribution of the input data. Here, we estimate 'yes-no' response bias in language models (LMs), with the goal of distinguishing the two theories, and explore two strategies for bias correction. We develop two yes-no question datasets derived from existing world knowledge datasets, and test 16 open-weight LMs. We find that LMs often show response bias on yes-no questions, but that it is highly variable, deviating from bias observed in humans. We further present a novel bias correction method, which eliminates bias and improves model performance. Evidence of non-humanlike response bias in LMs informs us on the source of yes-bias in humans, and the efficacy of our bias correction method holds promise for LM evaluation.",
    "language": "eng",
    "license": {
        "name": "",
        "short_name": "",
        "text": null,
        "url": ""
    },
    "keywords": [
        {
            "word": "Artificial Intelligence; Language and thought; Machine learning; Reasoning; Computational Modeling"
        }
    ],
    "section": "Papers with Poster Presentation",
    "is_remote": true,
    "remote_url": "https://escholarship.org/uc/item/2c04k26b",
    "frozenauthors": [
        {
            "first_name": "Om",
            "middle_name": "",
            "last_name": "Bhatt",
            "name_suffix": "",
            "institution": "Georgia Institute of Technology",
            "department": ""
        },
        {
            "first_name": "Anna",
            "middle_name": "",
            "last_name": "Ivanova",
            "name_suffix": "",
            "institution": "Georgia Institute of Technology",
            "department": ""
        }
    ],
    "date_submitted": null,
    "date_accepted": null,
    "date_published": "2025-01-01T13:00:00-05:00",
    "render_galley": null,
    "galleys": [
        {
            "label": "PDF",
            "type": "pdf",
            "path": "https://journalpub.escholarship.org/cognitivesciencesociety/article/49671/galley/37633/download/"
        }
    ]
}