Article Instance

HTTP 200 OK
Allow: GET
Content-Type: application/json
Vary: Accept

{
    "pk": 50051,
    "title": "SpikeBERT: A Language Understanding Spiking Neural Network Learned from BERT with Knowledge Distillation",
    "subtitle": null,
    "abstract": "Spiking neural networks (SNNs) provide a biologically inspired approach to deep learning, yet existing SNN models for language tasks remain simplistic and shallow, limiting their simulation of complex brain activity. This work investigates how deep SNNs process language, potentially advancing understanding of human language cognition. We introduce SpikeBERT, a pure SNN architecture adapted from spiking Transformers for text processing, coupled with a novel two-stage knowledge distillation method. First, SpikeBERT is pre-trained via knowledge distillation from BERT using unlabeled text. Second, task-specific fine-tuning occurs by distilling knowledge from BERT models trained on labeled data. Experiments demonstrate SpikeBERT outperforms state-of-the-art SNNs and achieves BERT-level performance on language understanding tasks with significantly lower energy consumption. The study bridges computational neuroscience and AI, offering insights into neuromorphic mechanisms of language processing. This energy-efficient framework advances SNN applications in NLP while providing a computational model to explore biological language cognition.",
    "language": "eng",
    "license": {
        "name": "",
        "short_name": "",
        "text": null,
        "url": ""
    },
    "keywords": [
        {
            "word": "Neuroscience; Language understanding; Machine learning; Computer-based experiment; Neural Networks"
        }
    ],
    "section": "Abstracts with Poster Presentation (accepted as Abstracts)",
    "is_remote": true,
    "remote_url": "https://escholarship.org/uc/item/0c3578sj",
    "frozenauthors": [
        {
            "first_name": "Changze",
            "middle_name": "",
            "last_name": "Lv",
            "name_suffix": "",
            "institution": "Fudan University",
            "department": ""
        },
        {
            "first_name": "Tianlong",
            "middle_name": "",
            "last_name": "Li",
            "name_suffix": "",
            "institution": "NLP Lab",
            "department": ""
        },
        {
            "first_name": "Xiaohua",
            "middle_name": "",
            "last_name": "Wang",
            "name_suffix": "",
            "institution": "Fudan University",
            "department": ""
        },
        {
            "first_name": "muling",
            "middle_name": "",
            "last_name": "wu",
            "name_suffix": "",
            "institution": "Fudan University",
            "department": ""
        },
        {
            "first_name": "Wenhao",
            "middle_name": "",
            "last_name": "Liu",
            "name_suffix": "",
            "institution": "Fudan University",
            "department": ""
        },
        {
            "first_name": "Shihan",
            "middle_name": "",
            "last_name": "Dou",
            "name_suffix": "",
            "institution": "Fudan University",
            "department": ""
        },
        {
            "first_name": "Xiaoqing",
            "middle_name": "",
            "last_name": "Zheng",
            "name_suffix": "",
            "institution": "Fudan University",
            "department": ""
        },
        {
            "first_name": "Xuanjing",
            "middle_name": "",
            "last_name": "Huang",
            "name_suffix": "",
            "institution": "Fudan University",
            "department": ""
        }
    ],
    "date_submitted": null,
    "date_accepted": null,
    "date_published": "2025-01-01T18:00:00Z",
    "render_galley": null,
    "galleys": [
        {
            "label": "PDF",
            "type": "pdf",
            "path": "https://journalpub.escholarship.org/cognitivesciencesociety/article/50051/galley/38013/download/"
        }
    ]
}