{
  "id": "2502.21074",
  "title": "CODI: Compressing Chain-of-Thought into Continuous Space via Self-Distillation",
  "authors": "Zhenyi Shen, Hanqi Yan, Linhai Zhang, Zhanghao Hu, Yali Du, Yulan He",
  "first_public": "2025-02-28",
  "venue": "EMNLP 2025",
  "arxiv": "2502.21074",
  "paper_url": "https://arxiv.org/abs/2502.21074",
  "lens": [
    "continuous",
    "internalized"
  ],
  "representation": [
    "continuous-state"
  ],
  "computation": [
    "token-wise"
  ],
  "training": [
    "self-distillation"
  ],
  "questions": [
    "capability",
    "efficiency"
  ],
  "evidence": [
    "empirical"
  ],
  "modality": [
    "language",
    "symbolic"
  ],
  "summary": "Uses self-distillation to align continuous internal reasoning states with explicit chain-of-thought behavior while reducing visible reasoning tokens.",
  "provenance": "Awesome-Latent-CoT",
  "author_names": [
    "Zhenyi Shen",
    "Hanqi Yan",
    "Linhai Zhang",
    "Zhanghao Hu",
    "Yali Du",
    "Yulan He"
  ],
  "record_url": "https://latent-implicit-thinking.github.io/papers/2502.21074/",
  "legacy_record_url": "https://latent-implicit-thinking.github.io/papers/#paper-2502-21074",
  "citation_date": "2025-02-28",
  "json_url": "https://latent-implicit-thinking.github.io/papers/2502.21074/record.json",
  "markdown_url": "https://latent-implicit-thinking.github.io/papers/2502.21074/index.md",
  "bibtex_url": "https://latent-implicit-thinking.github.io/papers/2502.21074/citation.bib",
  "csl_url": "https://latent-implicit-thinking.github.io/papers/2502.21074/citation.csl.json",
  "publication": {
    "title": "CODI: Compressing Chain-of-Thought into Continuous Space via Self-Distillation",
    "author_names": [
      "Zhenyi Shen",
      "Hanqi Yan",
      "Linhai Zhang",
      "Zhanghao Hu",
      "Yali Du",
      "Yulan He"
    ],
    "year": 2025,
    "type": "paper-conference",
    "container_title": "Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing",
    "publisher": "Association for Computational Linguistics",
    "doi": "10.18653/v1/2025.emnlp-main.36",
    "pages": "677-693",
    "url": "https://aclanthology.org/2025.emnlp-main.36/",
    "source_url": "https://aclanthology.org/2025.emnlp-main.36/",
    "verified_at": "2026-10-18"
  },
  "publication_bibtex_url": "https://latent-implicit-thinking.github.io/papers/2502.21074/publication.bib",
  "publication_csl_url": "https://latent-implicit-thinking.github.io/papers/2502.21074/publication.csl.json",
  "method_comparisons": [
    {
      "title": "Hidden-state distillation: the 2023 method and CODI",
      "text": "The 2023 implicit CoT paper proposes reasoning through hidden states and implements it with a teacher, a thought emulator and an answer-generating student. CODI also transfers explicit reasoning through hidden-state supervision, but uses one model for both teacher and student tasks. It aligns their states just before the answer, while the student takes successive continuous reasoning steps.",
      "source_ids": [
        "2311.01460",
        "2502.21074"
      ],
      "source_urls": [
        "https://arxiv.org/html/2311.01460v1#S2.SS3",
        "https://arxiv.org/html/2311.01460v1#S3",
        "https://arxiv.org/html/2502.21074v3#S3"
      ],
      "source_labels": [
        "Deng et al. (2023), §2.3",
        "Deng et al. (2023), §3",
        "CODI, §3"
      ],
      "topic_id": "internalized",
      "topic_label": "Internalized reasoning",
      "topic_url": "https://latent-implicit-thinking.github.io/topics/internalized-reasoning/"
    },
    {
      "title": "Coconut and CODI: similar computation, different training",
      "text": "Coconut and CODI both pass continuous states between reasoning steps, but supervise them differently. Coconut gradually replaces written steps with hidden-state inputs and trains on the remaining text. CODI jointly trains explicit and implicit reasoning tasks, matching their hidden states at a selected position before the answer. These models avoid decoding a word at every intermediate step; they still perform successive model passes, so fewer written tokens does not mean no reasoning-time computation.",
      "source_ids": [
        "2412.06769",
        "2502.21074"
      ],
      "source_urls": [
        "https://arxiv.org/html/2412.06769v3#S3",
        "https://arxiv.org/html/2502.21074v3#S3"
      ],
      "source_labels": [
        "Coconut, §3",
        "CODI, §3"
      ],
      "topic_id": "continuous",
      "topic_label": "Continuous-state reasoning",
      "topic_url": "https://latent-implicit-thinking.github.io/topics/continuous-states/"
    }
  ]
}