{
  "id": "2504.14992",
  "title": "Efficient Pretraining Length Scaling",
  "authors": "Bohong Wu, Shen Yan, Sijun Zhang, Jianqiao Lu, Yutao Zeng, Ya Wang, Xun Zhou",
  "first_public": "2025-04-21",
  "venue": "arXiv",
  "arxiv": "2504.14992",
  "paper_url": "https://arxiv.org/abs/2504.14992",
  "lens": [
    "learned-tokens",
    "recurrent"
  ],
  "representation": [
    "hidden-state",
    "cache-memory"
  ],
  "computation": [
    "token-wise",
    "stateful"
  ],
  "training": [
    "pretraining"
  ],
  "questions": [
    "efficiency",
    "scaling",
    "architecture"
  ],
  "evidence": [
    "empirical",
    "scaling"
  ],
  "modality": [
    "language"
  ],
  "summary": "Adds hidden decoding tokens during pretraining, then discards their key/value cache entries after use. Keeps the original tokens' entries to preserve longer-range context.",
  "provenance": "Primary-source literature search, 2026-09-04",
  "author_names": [
    "Bohong Wu",
    "Shen Yan",
    "Sijun Zhang",
    "Jianqiao Lu",
    "Yutao Zeng",
    "Ya Wang",
    "Xun Zhou"
  ],
  "record_url": "https://latent-implicit-thinking.github.io/papers/2504.14992/",
  "legacy_record_url": "https://latent-implicit-thinking.github.io/papers/#paper-2504-14992",
  "citation_date": "2025-04-21",
  "json_url": "https://latent-implicit-thinking.github.io/papers/2504.14992/record.json",
  "markdown_url": "https://latent-implicit-thinking.github.io/papers/2504.14992/index.md",
  "bibtex_url": "https://latent-implicit-thinking.github.io/papers/2504.14992/citation.bib",
  "csl_url": "https://latent-implicit-thinking.github.io/papers/2504.14992/citation.csl.json"
}