{
  "type": "tweet",
  "id": "2010660969719165133",
  "url": "https://x.com/SakanaAILabs/status/2010660969719165133",
  "twitterUrl": "https://twitter.com/SakanaAILabs/status/2010660969719165133",
  "text": "Extending the Context of Pretrained LLMs by Dropping Their Positional Embeddings\n\nhttps://t.co/Fp5IJS4LIC\n\nAuthors: Yoav Gelberg, Koshi Eguchi, Takuya Akiba (@iwiwi), and Edoardo Cetin.\n\nIn the paper, we provide a theoretical analysis of this trade-off. We prove that while NoPE transformers struggle to learn attention patterns from scratch due to vanishing gradients, standard RoPE scaling inevitably warps semantic attention by compressing low frequencies.\n\nDroPE sidesteps both issues: utilizing RoPE for convergence, then removing it to avoid semantic shift during extension.",
  "source": "Twitter for iPhone",
  "retweetCount": 12,
  "replyCount": 1,
  "likeCount": 128,
  "quoteCount": 4,
  "viewCount": 40560,
  "createdAt": "Mon Jan 12 10:31:46 +0000 2026",
  "lang": "en",
  "bookmarkCount": 58,
  "isReply": true,
  "inReplyToId": "2010508366574186825",
  "conversationId": "2010508366574186825",
  "displayTextRange": [
    0,
    303
  ],
  "inReplyToUserId": null,
  "inReplyToUsername": null,
  "author": {
    "type": "user",
    "userName": "SakanaAILabs",
    "url": "https://x.com/SakanaAILabs",
    "twitterUrl": "https://twitter.com/SakanaAILabs",
    "id": "218811492",
    "name": "Sakana AI",
    "isVerified": false,
    "isBlueVerified": true,
    "verifiedType": "Business",
    "profilePicture": "https://pbs.twimg.com/profile_images/1885939209388929024/dtnrOdGp_normal.jpg",
    "coverPicture": "https://pbs.twimg.com/profile_banners/218811492/1686643464",
    "description": "",
    "location": "Tokyo, Japan",
    "followers": 58629,
    "following": 0,
    "status": "",
    "canDm": false,
    "canMediaTag": true,
    "createdAt": "Tue Nov 23 10:20:07 +0000 2010",
    "entities": {
      "description": {
        "urls": []
      },
      "url": {}
    },
    "fastFollowersCount": 0,
    "favouritesCount": 0,
    "hasCustomTimelines": true,
    "isTranslator": false,
    "mediaCount": 291,
    "statusesCount": 789,
    "withheldInCountries": [],
    "affiliatesHighlightedLabel": {},
    "possiblySensitive": false,
    "pinnedTweetIds": [
      "1823178623513239992"
    ],
    "profile_bio": {
      "description": "We are building a world class AI R&D company in Tokyo. We want to develop AI solutions for Japan’s needs, and democratize AI in Japan. https://t.co/OWL1EpmoDP",
      "entities": {
        "description": {
          "urls": [
            {
              "display_url": "sakana.ai/careers",
              "expanded_url": "http://sakana.ai/careers",
              "indices": [
                135,
                158
              ],
              "url": "https://t.co/OWL1EpmoDP"
            }
          ]
        },
        "url": {
          "urls": [
            {
              "display_url": "sakana.ai",
              "expanded_url": "https://sakana.ai/",
              "indices": [
                0,
                23
              ],
              "url": "https://t.co/1m2lSgmHLu"
            }
          ]
        }
      }
    },
    "isAutomated": false,
    "automatedBy": null
  },
  "extendedEntities": {
    "media": [
      {
        "display_url": "pic.twitter.com/ZJ9GzqT58c",
        "expanded_url": "https://twitter.com/SakanaAILabs/status/2010660969719165133/photo/1",
        "ext_alt_text": "Expensive finetuning beyond pretraining sequence length has traditionally been required to effectively extend language model (LM) context. We break this bottleneck by Dropping the Positional Embeddings of LMs after training (DroPE). This method relies on three observations: First, positional embeddings (PEs) provide crucial inductive bias aiding pretraining convergence. Second, over-reliance on PEs prevents generalization to unseen lengths, even with scaling methods. Third, PEs are not inherently required for effective modeling and can be removed after a short recalibration. Empirically, DroPE yields seamless zero-shot context extension without long-context finetuning, preserving original capabilities. Our findings hold across different models and sizes, outperforming specialized architectures and established rotary PE scaling methods.",
        "ext_media_availability": {
          "status": "Available"
        },
        "features": {
          "all": {
            "tags": [
              {
                "name": "Yoav Gelberg",
                "screen_name": "yoav_gelberg",
                "type": "user",
                "user_id": "1520701329764663297"
              },
              {
                "name": "Edoardo Cetin",
                "screen_name": "edo_cet",
                "type": "user",
                "user_id": "1520719228264976384"
              }
            ]
          },
          "large": {
            "faces": [
              {
                "h": 119,
                "w": 119,
                "x": 727,
                "y": 1267
              },
              {
                "h": 155,
                "w": 155,
                "x": 829,
                "y": 1251
              }
            ]
          },
          "orig": {
            "faces": [
              {
                "h": 199,
                "w": 199,
                "x": 1210,
                "y": 2107
              },
              {
                "h": 259,
                "w": 259,
                "x": 1379,
                "y": 2080
              }
            ]
          }
        },
        "id_str": "2010657489755525120",
        "indices": [
          304,
          327
        ],
        "media_key": "3_2010657489755525120",
        "media_results": {
          "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARvnSlWB21AACgACG+dNf7/acM0AAA==",
          "result": {
            "__typename": "ApiMedia",
            "id": "QXBpTWVkaWE6DAABCgABG+dKVYHbUAAKAAIb501/v9pwzQAA",
            "media_key": "3_2010657489755525120"
          }
        },
        "media_url_https": "https://pbs.twimg.com/media/G-dKVYHbUAAC63Z.jpg",
        "original_info": {
          "focus_rects": [
            {
              "h": 1389,
              "w": 2480,
              "x": 0,
              "y": 0
            },
            {
              "h": 2480,
              "w": 2480,
              "x": 0,
              "y": 0
            },
            {
              "h": 2827,
              "w": 2480,
              "x": 0,
              "y": 0
            },
            {
              "h": 3404,
              "w": 1702,
              "x": 0,
              "y": 0
            },
            {
              "h": 3404,
              "w": 2480,
              "x": 0,
              "y": 0
            }
          ],
          "height": 3404,
          "width": 2480
        },
        "sizes": {
          "large": {
            "h": 2048,
            "w": 1492
          }
        },
        "type": "photo",
        "url": "https://t.co/ZJ9GzqT58c"
      }
    ]
  },
  "card": null,
  "place": {},
  "entities": {
    "urls": [
      {
        "display_url": "arxiv.org/abs/2512.12167",
        "expanded_url": "https://arxiv.org/abs/2512.12167",
        "indices": [
          82,
          105
        ],
        "url": "https://t.co/Fp5IJS4LIC"
      }
    ],
    "user_mentions": [
      {
        "id_str": "16555745",
        "indices": [
          158,
          164
        ],
        "name": "Takuya Akiba",
        "screen_name": "iwiwi"
      }
    ]
  },
  "quoted_tweet": null,
  "retweeted_tweet": null,
  "isLimitedReply": false,
  "article": null
}