Article Instance

HTTP 200 OK
Allow: GET
Content-Type: application/json
Vary: Accept

{
    "pk": 50396,
    "title": "Human and LLM performance on linguistic test: Content effect and task demands",
    "subtitle": null,
    "abstract": "Large Language Models (LLMs) display an impressive set of capabilities in linguistic understanding. While advanced models outperform humans on certain tasks, LLM reasoning and linguistic competency differs from that of humans (Felin & Holweg, 2024; Mahowald et al., 2024; Niu et al., 2024). In this study, we evaluate humans and GPT-4o on the Winograd Schema Challenge, a pronoun resolution task. We focus on Japanese, a relatively understudied language in the emergent field of human-LLM evaluation. To assess human vs. LLM performance, we manipulate task demands and content. We report three findings: (i) Humans outperform LLMs in the baseline condition, i.e. the standard pronoun resolution task. (ii) As task demands increase, both human and LLM performance on the task declines (cf. Hu & Frank, 2024). (iii) We find evidence for content effects (cf. Lampinen et al., 2024): LLMs surpass humans as the content of the task is manipulated to favor LLMs.",
    "language": "eng",
    "license": {
        "name": "",
        "short_name": "",
        "text": null,
        "url": ""
    },
    "keywords": [
        {
            "word": "Linguistics; Behavioral Science; Language understanding; Comparative Studies"
        }
    ],
    "section": "Member Abstracts with Poster Presentation",
    "is_remote": true,
    "remote_url": "https://escholarship.org/uc/item/13c146qr",
    "frozenauthors": [
        {
            "first_name": "May",
            "middle_name": "Lynn",
            "last_name": "Reese",
            "name_suffix": "",
            "institution": "San Francisco State University",
            "department": ""
        },
        {
            "first_name": "Anastasia",
            "middle_name": "",
            "last_name": "Smirnova",
            "name_suffix": "",
            "institution": "San Francisco State University",
            "department": ""
        }
    ],
    "date_submitted": null,
    "date_accepted": null,
    "date_published": "2025-01-01T18:00:00Z",
    "render_galley": null,
    "galleys": [
        {
            "label": "PDF",
            "type": "pdf",
            "path": "https://journalpub.escholarship.org/cognitivesciencesociety/article/50396/galley/38358/download/"
        }
    ]
}