{
  "id": "2310.02226",
  "title": "Think before you speak: Training language models with pause tokens",
  "authors": "Sachin Goyal, Ziwei Ji, Ankit Singh Rawat, Aditya Krishna Menon, Sanjiv Kumar, Vaishnavh Nagarajan",
  "first_public": "2023-10-03",
  "venue": "ICLR 2024",
  "arxiv": "2310.02226",
  "paper_url": "https://arxiv.org/abs/2310.02226",
  "lens": [
    "learned-tokens"
  ],
  "representation": [
    "discrete-token"
  ],
  "computation": [
    "token-wise"
  ],
  "training": [
    "pretraining"
  ],
  "questions": [
    "capability"
  ],
  "evidence": [
    "empirical"
  ],
  "modality": [
    "language"
  ],
  "summary": "Introduces pause tokens that give a language model additional computational positions before it must produce an answer.",
  "provenance": "Awesome-Latent-CoT",
  "author_names": [
    "Sachin Goyal",
    "Ziwei Ji",
    "Ankit Singh Rawat",
    "Aditya Krishna Menon",
    "Sanjiv Kumar",
    "Vaishnavh Nagarajan"
  ],
  "record_url": "https://latent-implicit-thinking.github.io/papers/2310.02226/",
  "legacy_record_url": "https://latent-implicit-thinking.github.io/papers/#paper-2310-02226",
  "citation_date": "2023-10-03",
  "json_url": "https://latent-implicit-thinking.github.io/papers/2310.02226/record.json",
  "markdown_url": "https://latent-implicit-thinking.github.io/papers/2310.02226/index.md",
  "bibtex_url": "https://latent-implicit-thinking.github.io/papers/2310.02226/citation.bib",
  "csl_url": "https://latent-implicit-thinking.github.io/papers/2310.02226/citation.csl.json",
  "publication": {
    "title": "Think before you speak: Training Language Models With Pause Tokens",
    "author_names": [
      "Sachin Goyal",
      "Ziwei Ji",
      "Ankit Singh Rawat",
      "Aditya Krishna Menon",
      "Sanjiv Kumar",
      "Vaishnavh Nagarajan"
    ],
    "year": 2024,
    "type": "paper-conference",
    "container_title": "International Conference on Learning Representations",
    "pages": "27896-27923",
    "volume": "2024",
    "url": "https://proceedings.iclr.cc/paper_files/paper/2024/hash/76917808731dae9e6d62c2a7a6afb542-Abstract-Conference.html",
    "source_url": "https://proceedings.iclr.cc/paper_files/paper/2024/hash/76917808731dae9e6d62c2a7a6afb542-Abstract-Conference.html",
    "verified_at": "2026-10-18"
  },
  "publication_bibtex_url": "https://latent-implicit-thinking.github.io/papers/2310.02226/publication.bib",
  "publication_csl_url": "https://latent-implicit-thinking.github.io/papers/2310.02226/publication.csl.json",
  "method_comparisons": [
    {
      "title": "Pausing and planning",
      "text": "Pause tokens add computation before the answer without choosing a different plan token for each step. Planning tokens instead select a learned token before generating each written reasoning step. The latter still produces a textual chain of thought. In the pause-token experiments, the training setup matters: gains were strongest when pauses were used in both pretraining and fine-tuning, not simply inserted into an unchanged model.",
      "source_ids": [
        "2310.02226",
        "2310.05707"
      ],
      "source_urls": [
        "https://proceedings.iclr.cc/paper_files/paper/2024/hash/76917808731dae9e6d62c2a7a6afb542-Abstract-Conference.html",
        "https://arxiv.org/html/2310.05707v3#S2"
      ],
      "source_labels": [
        "Pause-token paper",
        "Planning-token paper, §2"
      ],
      "topic_id": "learned-tokens",
      "topic_label": "Thinking tokens",
      "topic_url": "https://latent-implicit-thinking.github.io/topics/thinking-tokens/"
    }
  ]
}