Viewing enriched Twitter post
After part 3 of MoE 101 series we got two main questions: 1. why is MoE forward pass slower than dense network? 2. why can't I train 64 experts on a single GPU and hit OOM? we discuss both problems and solutions in part 4: https://t.co/uW6H78ZE56 1/n 🧵
{
"media": [
{
"url": "https://crmoxkoizveukayfjuyo.supabase.co/storage/v1/object/public/media/posts/1963690746323890376/media_0.jpg?",
"media_url": "https://crmoxkoizveukayfjuyo.supabase.co/storage/v1/object/public/media/posts/1963690746323890376/media_0.jpg?",
"type": "photo",
"filename": "media_0.jpg"
}
],
"processed_at": "2025-09-06T21:15:28.044609",
"pipeline_version": "2.0"
}{
"type": "tweet",
"id": "1963690746323890376",
"url": "https://x.com/dmsobol/status/1963690746323890376",
"twitterUrl": "https://twitter.com/dmsobol/status/1963690746323890376",
"text": "After part 3 of MoE 101 series we got two main questions: \n\n1. why is MoE forward pass slower than dense network? 2. why can't I train 64 experts on a single GPU and hit OOM? \n\nwe discuss both problems and solutions in part 4: https://t.co/uW6H78ZE56\n1/n 🧵",
"source": "Twitter for iPhone",
"retweetCount": 20,
"replyCount": 4,
"likeCount": 91,
"quoteCount": 1,
"viewCount": 11857,
"createdAt": "Thu Sep 04 19:48:52 +0000 2025",
"lang": "en",
"bookmarkCount": 73,
"isReply": false,
"inReplyToId": null,
"conversationId": "1963690746323890376",
"displayTextRange": [
0,
256
],
"inReplyToUserId": null,
"inReplyToUsername": null,
"author": {
"type": "user",
"userName": "dmsobol",
"url": "https://x.com/dmsobol",
"twitterUrl": "https://twitter.com/dmsobol",
"id": "1540768587488448512",
"name": "Daria Soboleva",
"isVerified": false,
"isBlueVerified": true,
"verifiedType": null,
"profilePicture": "https://pbs.twimg.com/profile_images/1780015722720292864/cYYAFmcb_normal.jpg",
"coverPicture": "https://pbs.twimg.com/profile_banners/1540768587488448512/1756093241",
"description": "",
"location": "Sunnyvale, California",
"followers": 861,
"following": 620,
"status": "",
"canDm": true,
"canMediaTag": true,
"createdAt": "Sat Jun 25 18:47:34 +0000 2022",
"entities": {
"description": {
"urls": []
},
"url": {}
},
"fastFollowersCount": 0,
"favouritesCount": 978,
"hasCustomTimelines": true,
"isTranslator": false,
"mediaCount": 32,
"statusesCount": 441,
"withheldInCountries": [],
"affiliatesHighlightedLabel": {},
"possiblySensitive": false,
"pinnedTweetIds": [
"1947729529050697808"
],
"profile_bio": {
"description": "Making MoE models work @Cerebras & posting about it | Creator of SlimPajama | Ex-@Google @Yandex @Cisco.",
"entities": {
"description": {
"user_mentions": [
{
"id_str": "0",
"indices": [
23,
32
],
"name": "",
"screen_name": "Cerebras"
},
{
"id_str": "0",
"indices": [
81,
88
],
"name": "",
"screen_name": "Google"
},
{
"id_str": "0",
"indices": [
89,
96
],
"name": "",
"screen_name": "Yandex"
},
{
"id_str": "0",
"indices": [
97,
103
],
"name": "",
"screen_name": "Cisco"
}
]
},
"url": {
"urls": [
{
"display_url": "soboleva-daria.github.io",
"expanded_url": "https://soboleva-daria.github.io/",
"indices": [
0,
23
],
"url": "https://t.co/n4YjHYsnpB"
}
]
}
}
},
"isAutomated": false,
"automatedBy": null
},
"extendedEntities": {},
"card": {
"binding_values": [
{
"key": "photo_image_full_size_large",
"value": {
"image_value": {
"height": 419,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=800x419",
"width": 800
}
}
},
{
"key": "thumbnail_image",
"value": {
"image_value": {
"height": 147,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=280x150",
"width": 280
}
}
},
{
"key": "description",
"value": {
"string_value": "Cerebras is the go-to platform for fast and effortless AI training. Learn more at cerebras.ai."
}
},
{
"key": "domain",
"value": {
"string_value": "www.cerebras.ai"
}
},
{
"key": "thumbnail_image_large",
"value": {
"image_value": {
"height": 315,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=600x600",
"width": 600
}
}
},
{
"key": "summary_photo_image_small",
"value": {
"image_value": {
"height": 202,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=386x202",
"width": 386
}
}
},
{
"key": "thumbnail_image_original",
"value": {
"image_value": {
"height": 630,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=orig",
"width": 1200
}
}
},
{
"key": "photo_image_full_size_small",
"value": {
"image_value": {
"height": 202,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=386x202",
"width": 386
}
}
},
{
"key": "summary_photo_image_large",
"value": {
"image_value": {
"height": 419,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=800x419",
"width": 800
}
}
},
{
"key": "thumbnail_image_small",
"value": {
"image_value": {
"height": 76,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=144x144",
"width": 144
}
}
},
{
"key": "thumbnail_image_x_large",
"value": {
"image_value": {
"height": 630,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=png&name=2048x2048_2_exp",
"width": 1200
}
}
},
{
"key": "photo_image_full_size_original",
"value": {
"image_value": {
"height": 630,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=orig",
"width": 1200
}
}
},
{
"key": "vanity_url",
"value": {
"scribe_key": "vanity_url",
"string_value": "cerebras.ai"
}
},
{
"key": "photo_image_full_size",
"value": {
"image_value": {
"height": 314,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=600x314",
"width": 600
}
}
},
{
"key": "thumbnail_image_color",
"value": {
"image_color_value": {
"palette": [
{
"percentage": 38.57,
"rgb": {
"blue": 1,
"green": 2,
"red": 25
}
},
{
"percentage": 35.21,
"rgb": {
"blue": 4,
"green": 48,
"red": 165
}
},
{
"percentage": 16.64,
"rgb": {
"blue": 1,
"green": 18,
"red": 83
}
},
{
"percentage": 3.42,
"rgb": {
"blue": 15,
"green": 93,
"red": 216
}
},
{
"percentage": 1.08,
"rgb": {
"blue": 2,
"green": 8,
"red": 51
}
}
]
}
}
},
{
"key": "title",
"value": {
"string_value": "Cerebras"
}
},
{
"key": "summary_photo_image_color",
"value": {
"image_color_value": {
"palette": [
{
"percentage": 38.57,
"rgb": {
"blue": 1,
"green": 2,
"red": 25
}
},
{
"percentage": 35.21,
"rgb": {
"blue": 4,
"green": 48,
"red": 165
}
},
{
"percentage": 16.64,
"rgb": {
"blue": 1,
"green": 18,
"red": 83
}
},
{
"percentage": 3.42,
"rgb": {
"blue": 15,
"green": 93,
"red": 216
}
},
{
"percentage": 1.08,
"rgb": {
"blue": 2,
"green": 8,
"red": 51
}
}
]
}
}
},
{
"key": "summary_photo_image_x_large",
"value": {
"image_value": {
"height": 630,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=png&name=2048x2048_2_exp",
"width": 1200
}
}
},
{
"key": "summary_photo_image",
"value": {
"image_value": {
"height": 314,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=600x314",
"width": 600
}
}
},
{
"key": "photo_image_full_size_color",
"value": {
"image_color_value": {
"palette": [
{
"percentage": 38.57,
"rgb": {
"blue": 1,
"green": 2,
"red": 25
}
},
{
"percentage": 35.21,
"rgb": {
"blue": 4,
"green": 48,
"red": 165
}
},
{
"percentage": 16.64,
"rgb": {
"blue": 1,
"green": 18,
"red": 83
}
},
{
"percentage": 3.42,
"rgb": {
"blue": 15,
"green": 93,
"red": 216
}
},
{
"percentage": 1.08,
"rgb": {
"blue": 2,
"green": 8,
"red": 51
}
}
]
}
}
},
{
"key": "photo_image_full_size_x_large",
"value": {
"image_value": {
"height": 630,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=png&name=2048x2048_2_exp",
"width": 1200
}
}
},
{
"key": "card_url",
"value": {
"scribe_key": "card_url",
"string_value": "https://t.co/uW6H78ZE56"
}
},
{
"key": "summary_photo_image_original",
"value": {
"image_value": {
"height": 630,
"url": "https://pbs.twimg.com/card_img/1963681405126459394/NB6SuxI-?format=jpg&name=orig",
"width": 1200
}
}
}
],
"card_platform": {
"platform": {
"audience": {
"name": "production"
},
"device": {
"name": "iPhone",
"version": "13"
}
}
},
"name": "summary_large_image",
"url": "https://t.co/uW6H78ZE56"
},
"place": {},
"entities": {
"urls": [
{
"display_url": "cerebras.ai/blog/moe-guide…",
"expanded_url": "https://www.cerebras.ai/blog/moe-guide-scale",
"indices": [
227,
250
],
"url": "https://t.co/uW6H78ZE56"
}
]
},
"quoted_tweet": {
"type": "tweet",
"id": "1963682001938194508",
"url": "https://x.com/CerebrasSystems/status/1963682001938194508",
"twitterUrl": "https://twitter.com/CerebrasSystems/status/1963682001938194508",
"text": "MoE 101 - Episode 4: Theoretical: 60% fewer FLOPs. Reality: 7x slow down \nYou followed all the tips from our last video. Your MoE model finally trains…\n\nThen you try to scale it on GPUs...and... memory issues ❌, underused experts ❌, unpredictable compute bottlenecks ❌.\n\nFrom expert parallelism on the GPU to BTA techniques on the Cerebras, @dmsobol breaks it down: https://t.co/v9MxR70NtH",
"source": "Twitter for iPhone",
"retweetCount": 3,
"replyCount": 1,
"likeCount": 80,
"quoteCount": 2,
"viewCount": 15910,
"createdAt": "Thu Sep 04 19:14:07 +0000 2025",
"lang": "en",
"bookmarkCount": 31,
"isReply": false,
"inReplyToId": null,
"conversationId": "1963682001938194508",
"displayTextRange": [
0,
276
],
"inReplyToUserId": null,
"inReplyToUsername": null,
"author": {
"type": "user",
"userName": "CerebrasSystems",
"url": "https://x.com/CerebrasSystems",
"twitterUrl": "https://twitter.com/CerebrasSystems",
"id": "751545566778171392",
"name": "Cerebras",
"isVerified": false,
"isBlueVerified": true,
"verifiedType": "Business",
"profilePicture": "https://pbs.twimg.com/profile_images/1894806880804913152/zLdh072p_normal.jpg",
"coverPicture": "https://pbs.twimg.com/profile_banners/751545566778171392/1747945971",
"description": "",
"location": "Sunnyvale, CA",
"followers": 34695,
"following": 255,
"status": "",
"canDm": true,
"canMediaTag": true,
"createdAt": "Fri Jul 08 22:36:41 +0000 2016",
"entities": {
"description": {
"urls": []
},
"url": {}
},
"fastFollowersCount": 0,
"favouritesCount": 1487,
"hasCustomTimelines": true,
"isTranslator": false,
"mediaCount": 1001,
"statusesCount": 2278,
"withheldInCountries": [],
"affiliatesHighlightedLabel": {},
"possiblySensitive": false,
"pinnedTweetIds": [
"1952785033024160096"
],
"profile_bio": {
"description": "The world's fastest AI inference and training.\nTry the latest open models at: https://t.co/jREGhLI2nj",
"entities": {
"description": {
"urls": [
{
"display_url": "inference.cerebras.ai",
"expanded_url": "https://inference.cerebras.ai/",
"indices": [
78,
101
],
"url": "https://t.co/jREGhLI2nj"
}
]
},
"url": {
"urls": [
{
"display_url": "cerebras.ai",
"expanded_url": "https://www.cerebras.ai/",
"indices": [
0,
23
],
"url": "https://t.co/SUREW9ZthN"
}
]
}
}
},
"isAutomated": false,
"automatedBy": null
},
"extendedEntities": {
"media": [
{
"additional_media_info": {
"monetizable": false
},
"display_url": "pic.twitter.com/b812dujd6S",
"expanded_url": "https://twitter.com/CerebrasSystems/status/1963682001938194508/video/1",
"ext_media_availability": {
"status": "Available"
},
"id_str": "1963678967271153664",
"indices": [
277,
300
],
"media_key": "13_1963678967271153664",
"media_results": {
"id": "QXBpTWVkaWFSZXN1bHRzOgwABAoAARtAY53hGmAAAAA=",
"result": {
"__typename": "ApiMedia",
"id": "QXBpTWVkaWE6DAAECgABG0BjneEaYAAAAA==",
"media_key": "13_1963678967271153664"
}
},
"media_url_https": "https://pbs.twimg.com/amplify_video_thumb/1963678967271153664/img/LDq05NpFdFKhE-0r.jpg",
"original_info": {
"focus_rects": [],
"height": 640,
"width": 1110
},
"sizes": {
"large": {
"h": 640,
"w": 1110
}
},
"type": "video",
"url": "https://t.co/b812dujd6S",
"video_info": {
"aspect_ratio": [
111,
64
],
"duration_millis": 119900,
"variants": [
{
"content_type": "application/x-mpegURL",
"url": "https://video-s.twimg.com/amplify_video/1963678967271153664/pl/876GjhfhHbB5Ft3H.m3u8?tag=21&v=1ab"
},
{
"bitrate": 256000,
"content_type": "video/mp4",
"url": "https://video-s.twimg.com/amplify_video/1963678967271153664/vid/avc1/468x270/eyiweHC7muWotSxn.mp4?tag=21"
},
{
"bitrate": 832000,
"content_type": "video/mp4",
"url": "https://video-s.twimg.com/amplify_video/1963678967271153664/vid/avc1/624x360/odZtCeWqKrjxRq6R.mp4?tag=21"
},
{
"bitrate": 2176000,
"content_type": "video/mp4",
"url": "https://video-s.twimg.com/amplify_video/1963678967271153664/vid/avc1/1110x640/HdZPVf2mlCefBufQ.mp4?tag=21"
}
]
}
}
]
},
"card": null,
"place": {},
"entities": {
"urls": [
{
"display_url": "cerebras.ai/blog/moe-guide…",
"expanded_url": "https://www.cerebras.ai/blog/moe-guide-scale",
"indices": [
367,
390
],
"url": "https://t.co/v9MxR70NtH"
}
],
"user_mentions": [
{
"id_str": "1540768587488448512",
"indices": [
342,
350
],
"name": "Daria Soboleva",
"screen_name": "dmsobol"
}
]
},
"quoted_tweet": null,
"retweeted_tweet": null,
"isLimitedReply": false,
"article": null
},
"retweeted_tweet": null,
"isLimitedReply": false,
"article": null
}