Article Instance

HTTP 200 OK
Allow: GET
Content-Type: application/json
Vary: Accept

{
    "pk": 49744,
    "title": "Extending a Mathematical Theory of the Emergence of Knowledge from the Experience to Capture Learning Dynamics in Transformers",
    "subtitle": null,
    "abstract": "The Transformer architecture used in LLMs has garnered\nwidespread attention due to these model's human-like\nconceptual knowledge and language understanding, yet\nunderstanding how these models' capabilities result from\nexperience-guided learning, and connecting this learning\nprocess with the structure in their training data, can seem\nintractable. Here we present preliminary steps to\ncharacterizing the developmental trajectory of a minimal\nTransformer trained on a next-token prediction task, using a\nsimple dataset with quantifiable uncertainty and a simple,\nintuitively characterizable structure that captures some aspects\nof natural semantic structure learned by LLMs from large\ndatasets. We show how the dynamic learning process of this\nmodel is a predictable consequence of the structure of the\ntraining data, exhibiting attested features of human semantic\ndevelopment, as captured in a theory of neural network\nlearning dynamics (Saxe et. al. 2019) previously used to\ncapture such dynamics in a network originally introduced by\nRumelhart & Todd (1993).",
    "language": "eng",
    "license": {
        "name": "",
        "short_name": "",
        "text": null,
        "url": ""
    },
    "keywords": [
        {
            "word": "Cognitive architectures; Cognitive development; Machine learning; Neural Networks"
        }
    ],
    "section": "Papers with Poster Presentation",
    "is_remote": true,
    "remote_url": "https://escholarship.org/uc/item/8jb4v2wd",
    "frozenauthors": [
        {
            "first_name": "Sabrina",
            "middle_name": "",
            "last_name": "Jones",
            "name_suffix": "",
            "institution": "Stanford University",
            "department": ""
        },
        {
            "first_name": "Jay",
            "middle_name": "",
            "last_name": "McClelland",
            "name_suffix": "",
            "institution": "Stanford University",
            "department": ""
        }
    ],
    "date_submitted": null,
    "date_accepted": null,
    "date_published": "2025-01-01T18:00:00Z",
    "render_galley": null,
    "galleys": [
        {
            "label": "PDF",
            "type": "pdf",
            "path": "https://journalpub.escholarship.org/cognitivesciencesociety/article/49744/galley/37706/download/"
        }
    ]
}