Viewing enriched Twitter post
Our findings: 1) For short seqs, increasing density or size provides gains. For long seqs, high sparsity performs best. 2) Higher sparsity is possible for decoding and larger models. However, most configs deteriorate performance significantly for at least one task. 3) There is no single sparse attention method that excels across all tasks. 4) We establish scaling laws based on model size, sequence length, compression ratio, and task id that reliably predict performance.
{
"media": [
{
"type": "photo",
"url": "https://crmoxkoizveukayfjuyo.supabase.co/storage/v1/object/public/media/posts/1916853646253346915/media_0.jpg?",
"filename": "media_0.jpg"
}
],
"processed_at": "2025-08-15T23:25:34.994176",
"pipeline_version": "2.0"
}{
"type": "tweet",
"id": "1916853646253346915",
"url": "https://x.com/seb_ruder/status/1916853646253346915",
"twitterUrl": "https://twitter.com/seb_ruder/status/1916853646253346915",
"text": "Our findings:\n1) For short seqs, increasing density or size provides gains. For long seqs, high sparsity performs best.\n2) Higher sparsity is possible for decoding and larger models. However, most configs deteriorate performance significantly for at least one task.\n3) There is no single sparse attention method that excels across all tasks.\n4) We establish scaling laws based on model size, sequence length, compression ratio, and task id that reliably predict performance.",
"source": "Twitter for iPhone",
"retweetCount": 0,
"replyCount": 1,
"likeCount": 5,
"quoteCount": 0,
"viewCount": 1637,
"createdAt": "Mon Apr 28 13:54:57 +0000 2025",
"lang": "en",
"bookmarkCount": 0,
"isReply": true,
"inReplyToId": "1916850948741239173",
"conversationId": "1916850948741239173",
"displayTextRange": [
0,
281
],
"inReplyToUserId": "2785337469",
"inReplyToUsername": "seb_ruder",
"author": {
"type": "user",
"userName": "seb_ruder",
"url": "https://x.com/seb_ruder",
"twitterUrl": "https://twitter.com/seb_ruder",
"id": "2785337469",
"name": "Sebastian Ruder @ ACL",
"isVerified": false,
"isBlueVerified": true,
"verifiedType": null,
"profilePicture": "https://pbs.twimg.com/profile_images/1100429844830146560/mgKi3vLm_normal.jpg",
"coverPicture": "https://pbs.twimg.com/profile_banners/2785337469/1438539641",
"description": "",
"location": "Berlin, Deutschland",
"followers": 92002,
"following": 1458,
"status": "",
"canDm": false,
"canMediaTag": true,
"createdAt": "Fri Sep 26 12:47:34 +0000 2014",
"entities": {
"description": {
"urls": []
},
"url": {}
},
"fastFollowersCount": 0,
"favouritesCount": 9895,
"hasCustomTimelines": true,
"isTranslator": true,
"mediaCount": 380,
"statusesCount": 4264,
"withheldInCountries": [],
"affiliatesHighlightedLabel": {},
"possiblySensitive": false,
"pinnedTweetIds": [],
"profile_bio": {
"description": "Research Scientist @AIatMeta • Ex @Cohere @GoogleDeepMind",
"entities": {
"description": {
"user_mentions": [
{
"id_str": "0",
"indices": [
19,
28
],
"name": "",
"screen_name": "AIatMeta"
},
{
"id_str": "0",
"indices": [
34,
41
],
"name": "",
"screen_name": "Cohere"
},
{
"id_str": "0",
"indices": [
42,
57
],
"name": "",
"screen_name": "GoogleDeepMind"
}
]
},
"url": {
"urls": [
{
"display_url": "ruder.io",
"expanded_url": "http://www.ruder.io",
"indices": [
0,
23
],
"url": "https://t.co/1GdhVCfGj9"
}
]
}
}
},
"isAutomated": false,
"automatedBy": null
},
"extendedEntities": {
"media": [
{
"allow_download_status": {
"allow_download": true
},
"display_url": "pic.twitter.com/ohNRYwFLDE",
"expanded_url": "https://twitter.com/seb_ruder/status/1916853646253346915/photo/1",
"ext_media_availability": {
"status": "Available"
},
"features": {
"large": {},
"orig": {}
},
"id_str": "1916853606063489024",
"indices": [
282,
305
],
"media_key": "3_1916853606063489024",
"media_results": {
"id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARqaCDLj10AACgACGpoIPD9X0GMAAA==",
"result": {
"__typename": "ApiMedia",
"id": "QXBpTWVkaWE6DAABCgABGpoIMuPXQAAKAAIamgg8P1fQYwAA",
"media_key": "3_1916853606063489024"
}
},
"media_url_https": "https://pbs.twimg.com/media/GpoIMuPXQAA8K8B.jpg",
"original_info": {
"focus_rects": [
{
"h": 1039,
"w": 1856,
"x": 0,
"y": 732
},
{
"h": 1856,
"w": 1856,
"x": 0,
"y": 323
},
{
"h": 2116,
"w": 1856,
"x": 0,
"y": 193
},
{
"h": 2384,
"w": 1192,
"x": 417,
"y": 0
},
{
"h": 2384,
"w": 1856,
"x": 0,
"y": 0
}
],
"height": 2384,
"width": 1856
},
"sizes": {
"large": {
"h": 2048,
"w": 1594
}
},
"type": "photo",
"url": "https://t.co/ohNRYwFLDE"
}
]
},
"card": null,
"place": {},
"entities": {},
"quoted_tweet": null,
"retweeted_tweet": null,
"article": null
}