{
  "id": "2502.08524",
  "title": "LLM Pretraining with Continuous Concepts",
  "authors": "Jihoon Tack, Jack Lanchantin, Jane Yu, Andrew Cohen, Ilia Kulikov, Janice Lan, Shibo Hao, Yuandong Tian, Jason Weston, Xian Li",
  "first_public": "2025-02-12",
  "venue": "arXiv",
  "arxiv": "2502.08524",
  "paper_url": "https://arxiv.org/abs/2502.08524",
  "code_url": "https://github.com/facebookresearch/RAM/tree/main/projects/cocomix",
  "lens": [
    "continuous"
  ],
  "representation": [
    "continuous-state"
  ],
  "computation": [
    "token-wise"
  ],
  "training": [
    "pretraining"
  ],
  "questions": [
    "capability",
    "efficiency"
  ],
  "evidence": [
    "empirical"
  ],
  "modality": [
    "language"
  ],
  "summary": "Interleaves continuous concept representations with text during pretraining so language models learn to compute over concepts as well as tokens.",
  "provenance": "Awesome-Latent-CoT",
  "author_names": [
    "Jihoon Tack",
    "Jack Lanchantin",
    "Jane Yu",
    "Andrew Cohen",
    "Ilia Kulikov",
    "Janice Lan",
    "Shibo Hao",
    "Yuandong Tian",
    "Jason Weston",
    "Xian Li"
  ],
  "record_url": "https://latent-implicit-thinking.github.io/papers/2502.08524/",
  "legacy_record_url": "https://latent-implicit-thinking.github.io/papers/#paper-2502-08524",
  "citation_date": "2025-02-12",
  "json_url": "https://latent-implicit-thinking.github.io/papers/2502.08524/record.json",
  "markdown_url": "https://latent-implicit-thinking.github.io/papers/2502.08524/index.md",
  "bibtex_url": "https://latent-implicit-thinking.github.io/papers/2502.08524/citation.bib",
  "csl_url": "https://latent-implicit-thinking.github.io/papers/2502.08524/citation.csl.json"
}