🐦 Twitter Post Details

Viewing enriched Twitter post

@NousResearch

Today we release Token Superposition Training (TST), a modification to the standard LLM pretraining loop that produces a 2-3× wall-clock speedup at matched FLOPs without changing the model architecture, optimizer, tokenizer, or training data. During the first third of training, the model reads and predicts contiguous bags of tokens, averaging their embeddings on the input side and predicting the next bag with a modified cross-entropy on the output side. For the remainder of the run, it trains normally on next-token prediction. The inference-time model is identical to one produced by conventional pretraining. Validated at 270M, 600M, and 3B dense scales, and at 10B-A1B MoE. The work on TST was led by @bloc97_, @gigant_theo, and @theemozilla.

Media 1
View on Twitter

📊 Media Metadata

{
  "media": [
    {
      "url": "https://crmoxkoizveukayfjuyo.supabase.co/storage/v1/object/public/media/posts/2054610062836892054/media_0.jpg",
      "media_url": "https://crmoxkoizveukayfjuyo.supabase.co/storage/v1/object/public/media/posts/2054610062836892054/media_0.jpg",
      "type": "photo",
      "filename": "media_0.jpg"
    }
  ],
  "processed_at": "2026-05-13T18:50:06.788164",
  "pipeline_version": "2.0"
}

🔧 Raw API Response

{
  "type": "tweet",
  "id": "2054610062836892054",
  "url": "https://x.com/NousResearch/status/2054610062836892054",
  "twitterUrl": "https://twitter.com/NousResearch/status/2054610062836892054",
  "text": "Today we release Token Superposition Training (TST), a modification to the standard LLM pretraining loop that produces a 2-3× wall-clock speedup at matched FLOPs without changing the model architecture, optimizer, tokenizer, or training data.\n\nDuring the first third of training, the model reads and predicts contiguous bags of tokens, averaging their embeddings on the input side and predicting the next bag with a modified cross-entropy on the output side. For the remainder of the run, it trains normally on next-token prediction. The inference-time model is identical to one produced by conventional pretraining.\n\nValidated at 270M, 600M, and 3B dense scales, and at 10B-A1B MoE.\n\nThe work on TST was led by @bloc97_, @gigant_theo, and @theemozilla.",
  "source": "Twitter for iPhone",
  "retweetCount": 96,
  "replyCount": 55,
  "likeCount": 860,
  "quoteCount": 27,
  "viewCount": 38152,
  "createdAt": "Wed May 13 17:09:46 +0000 2026",
  "lang": "en",
  "bookmarkCount": 358,
  "isReply": false,
  "inReplyToId": null,
  "conversationId": "2054610062836892054",
  "displayTextRange": [
    0,
    279
  ],
  "inReplyToUserId": null,
  "inReplyToUsername": null,
  "author": {
    "type": "user",
    "userName": "NousResearch",
    "url": "https://x.com/NousResearch",
    "twitterUrl": "https://twitter.com/NousResearch",
    "id": "1318419526132862976",
    "name": "Nous Research",
    "isVerified": false,
    "isBlueVerified": true,
    "verifiedType": "Business",
    "profilePicture": "https://pbs.twimg.com/profile_images/1816254738234761216/TX7TW-Mp_normal.jpg",
    "coverPicture": "https://pbs.twimg.com/profile_banners/1318419526132862976/1698625831",
    "description": "",
    "location": "USA",
    "followers": 170316,
    "following": 24,
    "status": "",
    "canDm": true,
    "canMediaTag": true,
    "createdAt": "Tue Oct 20 05:11:31 +0000 2020",
    "entities": {
      "description": {
        "urls": []
      },
      "url": {}
    },
    "fastFollowersCount": 0,
    "favouritesCount": 15851,
    "hasCustomTimelines": true,
    "isTranslator": false,
    "mediaCount": 315,
    "statusesCount": 2542,
    "withheldInCountries": [],
    "affiliatesHighlightedLabel": {},
    "possiblySensitive": false,
    "pinnedTweetIds": [
      "2052904761087729897"
    ],
    "profile_bio": {
      "description": "World-class open source AI\nhttps://t.co/vrD0aDJeto",
      "entities": {
        "description": {
          "urls": [
            {
              "display_url": "discord.gg/nousresearch",
              "expanded_url": "https://discord.gg/nousresearch",
              "indices": [
                27,
                50
              ],
              "url": "https://t.co/vrD0aDJeto"
            }
          ]
        },
        "url": {
          "urls": [
            {
              "display_url": "hermes-agent.nousresearch.com",
              "expanded_url": "http://hermes-agent.nousresearch.com",
              "indices": [
                0,
                23
              ],
              "url": "https://t.co/WhkKIXKwT3"
            }
          ]
        }
      }
    },
    "isAutomated": false,
    "automatedBy": null
  },
  "extendedEntities": {
    "media": [
      {
        "allow_download_status": {
          "allow_download": true
        },
        "display_url": "pic.twitter.com/QIYV5bhWHe",
        "expanded_url": "https://twitter.com/NousResearch/status/2054610062836892054/photo/1",
        "ext_media_availability": {
          "status": "Available"
        },
        "features": {
          "large": {
            "faces": [
              {
                "h": 98,
                "w": 98,
                "x": 1394,
                "y": 642
              }
            ]
          },
          "orig": {
            "faces": [
              {
                "h": 196,
                "w": 196,
                "x": 2788,
                "y": 1284
              }
            ]
          }
        },
        "id_str": "2054601675059380224",
        "indices": [
          280,
          303
        ],
        "media_key": "3_2054601675059380224",
        "media_results": {
          "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARyDaVZ01kAACgACHINw92NWAZYAAA==",
          "result": {
            "__typename": "ApiMedia",
            "id": "QXBpTWVkaWE6DAABCgABHINpVnTWQAAKAAIcg3D3Y1YBlgAA",
            "media_key": "3_2054601675059380224"
          }
        },
        "media_url_https": "https://pbs.twimg.com/media/HINpVnTWQAAN82c.jpg",
        "original_info": {
          "focus_rects": [
            {
              "h": 2294,
              "w": 4096,
              "x": 0,
              "y": 0
            },
            {
              "h": 4096,
              "w": 4096,
              "x": 0,
              "y": 0
            },
            {
              "h": 4096,
              "w": 3593,
              "x": 252,
              "y": 0
            },
            {
              "h": 4096,
              "w": 2048,
              "x": 1024,
              "y": 0
            },
            {
              "h": 4096,
              "w": 4096,
              "x": 0,
              "y": 0
            }
          ],
          "height": 4096,
          "width": 4096
        },
        "sizes": {
          "large": {
            "h": 2048,
            "w": 2048
          }
        },
        "type": "photo",
        "url": "https://t.co/QIYV5bhWHe"
      }
    ]
  },
  "card": null,
  "place": {},
  "entities": {
    "hashtags": [],
    "symbols": [],
    "urls": [],
    "user_mentions": [
      {
        "id_str": "1699560720738787328",
        "indices": [
          712,
          720
        ],
        "name": "Bowen Peng",
        "screen_name": "bloc97_"
      },
      {
        "id_str": "1517256873791086594",
        "indices": [
          722,
          734
        ],
        "name": "Théo Gigant",
        "screen_name": "gigant_theo"
      },
      {
        "id_str": "17763307",
        "indices": [
          740,
          752
        ],
        "name": "emozilla",
        "screen_name": "theemozilla"
      }
    ]
  },
  "quoted_tweet": null,
  "retweeted_tweet": null,
  "isLimitedReply": false,
  "communityInfo": null,
  "article": null
}