Article Instance

HTTP 200 OK
Allow: GET
Content-Type: application/json
Vary: Accept

{
    "pk": 50211,
    "title": "Evaluating Vision Language Models Through Concept Hacking",
    "subtitle": null,
    "abstract": "Evaluating the cognitive abilities of Vision-Language Models (VLMs) is challenging due to their reliance on spurious correlations. To distinguish shortcut-taking from genuine reasoning, we introduce Concept Hacking, a paradigm manipulating concept-relevant information to flip the ground-truth but preserving concept-irrelevant confounds. For instance, in a perceptual constancy test, models must recognize that a uniformly wide bridge does not narrow in the distance; the manipulated condition using concept hacking altered the bridge to actually taper. We assessed 209 models across 45 experiment pairs spanning nine low-level cognitive abilities, encompassing all five core knowledge domains. Comparing performance on manipulated versus standard conditions revealed that models fell into shortcut-reliant or illusory understanding types, with none approaching human-level performance. Models of varying sizes appear in each category, indicating that scaling neither imparts core knowledge nor reduces shortcut reliance. These findings highlight fundamental limitations in current VLMs, reinforcing concerns about their ability to achieve genuine understanding.",
    "language": "eng",
    "license": {
        "name": "",
        "short_name": "",
        "text": null,
        "url": ""
    },
    "keywords": [
        {
            "word": "Computer Science; Psychology; Cognitive development; Concepts and categories; Machine learning"
        }
    ],
    "section": "Member Abstracts with Poster Presentation",
    "is_remote": true,
    "remote_url": "https://escholarship.org/uc/item/56k2j0br",
    "frozenauthors": [
        {
            "first_name": "Yijiang",
            "middle_name": "",
            "last_name": "Li",
            "name_suffix": "",
            "institution": "Electrical and computer engineering",
            "department": ""
        },
        {
            "first_name": "Bingyang",
            "middle_name": "",
            "last_name": "Wang",
            "name_suffix": "",
            "institution": "Emory University",
            "department": ""
        },
        {
            "first_name": "Tianwei",
            "middle_name": "",
            "last_name": "Zhao",
            "name_suffix": "",
            "institution": "Johns Hopkins University",
            "department": ""
        },
        {
            "first_name": "Qingying",
            "middle_name": "",
            "last_name": "Gao",
            "name_suffix": "",
            "institution": "Computer Science",
            "department": ""
        },
        {
            "first_name": "Hokin",
            "middle_name": "",
            "last_name": "Deng",
            "name_suffix": "",
            "institution": "Harvard University",
            "department": ""
        },
        {
            "first_name": "Dezhi",
            "middle_name": "",
            "last_name": "Luo",
            "name_suffix": "",
            "institution": "University of Michigan",
            "department": ""
        }
    ],
    "date_submitted": null,
    "date_accepted": null,
    "date_published": "2025-01-02T00:00:00+06:00",
    "render_galley": null,
    "galleys": [
        {
            "label": "PDF",
            "type": "pdf",
            "path": "https://journalpub.escholarship.org/cognitivesciencesociety/article/50211/galley/38173/download/"
        }
    ]
}