{
  "type": "tweet",
  "id": "2037663231511322831",
  "url": "https://x.com/levidiamode/status/2037663231511322831",
  "twitterUrl": "https://twitter.com/levidiamode/status/2037663231511322831",
  "text": "Day 83/365 of GPU Programming\n\nLooking at DeepSeek's Multi-Head Latent Attention today. The last part of the AMD challenge series is to optimize an MLA decode kernel for MI355X where  the absorbed Q and compressed KV cache are given and your task is to do the attention computation. \n\nA resource that really helped internalize what MLA does was @rasbt's incredible visual guide to attention variants in LLMs (luckily he posted that last week!), which covers everything from MHA to GQA to MLA to SWA, et cetera. If there's one place to get a visual intuition for recent attention mechanisms, it's this blog post. \n\n@jbhuang0604's video on MQA, GQA,MLA and DSA was the best conceptual intro I found on the topic and progressively builds up the ideas from first principles.\n\nThe Welch Labs analysis of MLA is a great watch as well. Beautiful visualization of the changes DeepSeek made for MLA. \n\nTried out a few kernels once I had a basic understanding of MLA and I think I'm slowly getting more comfortable with at least analyzing kernels.",
  "source": "Twitter for iPhone",
  "retweetCount": 28,
  "replyCount": 2,
  "likeCount": 284,
  "quoteCount": 0,
  "viewCount": 12244,
  "createdAt": "Fri Mar 27 22:49:07 +0000 2026",
  "lang": "en",
  "bookmarkCount": 176,
  "isReply": false,
  "inReplyToId": null,
  "conversationId": "2037663231511322831",
  "displayTextRange": [
    0,
    269
  ],
  "inReplyToUserId": null,
  "inReplyToUsername": null,
  "author": {
    "type": "user",
    "userName": "levidiamode",
    "url": "https://x.com/levidiamode",
    "twitterUrl": "https://twitter.com/levidiamode",
    "id": "1933932995528044544",
    "name": "levi",
    "isVerified": false,
    "isBlueVerified": true,
    "verifiedType": null,
    "profilePicture": "https://pbs.twimg.com/profile_images/2005322181538983936/KvzCyzRA_normal.jpg",
    "coverPicture": "https://pbs.twimg.com/profile_banners/1933932995528044544/1766941466",
    "description": "",
    "location": "out of memory",
    "followers": 1496,
    "following": 404,
    "status": "",
    "canDm": false,
    "canMediaTag": true,
    "createdAt": "Sat Jun 14 17:02:38 +0000 2025",
    "entities": {
      "description": {
        "urls": []
      },
      "url": {}
    },
    "fastFollowersCount": 0,
    "favouritesCount": 1105,
    "hasCustomTimelines": true,
    "isTranslator": false,
    "mediaCount": 88,
    "statusesCount": 223,
    "withheldInCountries": [],
    "affiliatesHighlightedLabel": {},
    "possiblySensitive": false,
    "pinnedTweetIds": [
      "2034732672845488598"
    ],
    "profile_bio": {
      "description": "365 days of GPU programming \n▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 83/365",
      "entities": {
        "description": {
          "hashtags": [],
          "symbols": [],
          "urls": [],
          "user_mentions": []
        },
        "url": {
          "urls": [
            {
              "display_url": "substack.com/@levidiamode",
              "expanded_url": "https://substack.com/@levidiamode",
              "indices": [
                0,
                23
              ],
              "url": "https://t.co/I4wYT0Z3IY"
            }
          ]
        }
      }
    },
    "isAutomated": false,
    "automatedBy": null
  },
  "extendedEntities": {
    "media": [
      {
        "allow_download_status": {
          "allow_download": true
        },
        "display_url": "pic.twitter.com/LtEjoVlpMJ",
        "expanded_url": "https://twitter.com/levidiamode/status/2037663231511322831/photo/1",
        "ext_media_availability": {
          "status": "Available"
        },
        "features": {
          "large": {
            "faces": []
          },
          "orig": {
            "faces": []
          }
        },
        "id_str": "2037657382583754752",
        "indices": [
          270,
          293
        ],
        "media_key": "3_2037657382583754752",
        "media_results": {
          "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARxHNpfu22AACgACHEc76b4bYM8AAA==",
          "result": {
            "__typename": "ApiMedia",
            "id": "QXBpTWVkaWE6DAABCgABHEc2l+7bYAAKAAIcRzvpvhtgzwAA",
            "media_key": "3_2037657382583754752"
          }
        },
        "media_url_https": "https://pbs.twimg.com/media/HEc2l-7bYAAu-I-.jpg",
        "original_info": {
          "focus_rects": [
            {
              "h": 858,
              "w": 1532,
              "x": 0,
              "y": 0
            },
            {
              "h": 858,
              "w": 858,
              "x": 152,
              "y": 0
            },
            {
              "h": 858,
              "w": 753,
              "x": 205,
              "y": 0
            },
            {
              "h": 858,
              "w": 429,
              "x": 367,
              "y": 0
            },
            {
              "h": 858,
              "w": 1550,
              "x": 0,
              "y": 0
            }
          ],
          "height": 858,
          "width": 1550
        },
        "sizes": {
          "large": {
            "h": 858,
            "w": 1550
          }
        },
        "type": "photo",
        "url": "https://t.co/LtEjoVlpMJ"
      },
      {
        "allow_download_status": {
          "allow_download": true
        },
        "display_url": "pic.twitter.com/LtEjoVlpMJ",
        "expanded_url": "https://twitter.com/levidiamode/status/2037663231511322831/photo/1",
        "ext_media_availability": {
          "status": "Available"
        },
        "features": {
          "large": {
            "faces": []
          },
          "orig": {
            "faces": []
          }
        },
        "id_str": "2037657549470846976",
        "indices": [
          270,
          293
        ],
        "media_key": "3_2037657549470846976",
        "media_results": {
          "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARxHNr7KGlAACgACHEc76b4bYM8AAA==",
          "result": {
            "__typename": "ApiMedia",
            "id": "QXBpTWVkaWE6DAABCgABHEc2vsoaUAAKAAIcRzvpvhtgzwAA",
            "media_key": "3_2037657549470846976"
          }
        },
        "media_url_https": "https://pbs.twimg.com/media/HEc2vsoaUAA3BMs.jpg",
        "original_info": {
          "focus_rects": [
            {
              "h": 1561,
              "w": 2787,
              "x": 0,
              "y": 542
            },
            {
              "h": 2731,
              "w": 2731,
              "x": 56,
              "y": 0
            },
            {
              "h": 2731,
              "w": 2396,
              "x": 391,
              "y": 0
            },
            {
              "h": 2731,
              "w": 1366,
              "x": 1421,
              "y": 0
            },
            {
              "h": 2731,
              "w": 2787,
              "x": 0,
              "y": 0
            }
          ],
          "height": 2731,
          "width": 2787
        },
        "sizes": {
          "large": {
            "h": 2007,
            "w": 2048
          }
        },
        "type": "photo",
        "url": "https://t.co/LtEjoVlpMJ"
      },
      {
        "allow_download_status": {
          "allow_download": true
        },
        "display_url": "pic.twitter.com/LtEjoVlpMJ",
        "expanded_url": "https://twitter.com/levidiamode/status/2037663231511322831/photo/1",
        "ext_media_availability": {
          "status": "Available"
        },
        "features": {
          "large": {
            "faces": []
          },
          "orig": {
            "faces": []
          }
        },
        "id_str": "2037661697503023104",
        "indices": [
          270,
          293
        ],
        "media_key": "3_2037661697503023104",
        "media_results": {
          "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARxHOoSUFlAACgACHEc76b4bYM8AAA==",
          "result": {
            "__typename": "ApiMedia",
            "id": "QXBpTWVkaWE6DAABCgABHEc6hJQWUAAKAAIcRzvpvhtgzwAA",
            "media_key": "3_2037661697503023104"
          }
        },
        "media_url_https": "https://pbs.twimg.com/media/HEc6hJQWUAA6Ssf.jpg",
        "original_info": {
          "focus_rects": [
            {
              "h": 1393,
              "w": 2488,
              "x": 66,
              "y": 0
            },
            {
              "h": 1393,
              "w": 1393,
              "x": 896,
              "y": 0
            },
            {
              "h": 1393,
              "w": 1222,
              "x": 981,
              "y": 0
            },
            {
              "h": 1393,
              "w": 697,
              "x": 1244,
              "y": 0
            },
            {
              "h": 1393,
              "w": 2554,
              "x": 0,
              "y": 0
            }
          ],
          "height": 1393,
          "width": 2554
        },
        "sizes": {
          "large": {
            "h": 1117,
            "w": 2048
          }
        },
        "type": "photo",
        "url": "https://t.co/LtEjoVlpMJ"
      },
      {
        "allow_download_status": {
          "allow_download": true
        },
        "display_url": "pic.twitter.com/LtEjoVlpMJ",
        "expanded_url": "https://twitter.com/levidiamode/status/2037663231511322831/photo/1",
        "ext_media_availability": {
          "status": "Available"
        },
        "features": {
          "large": {
            "faces": []
          },
          "orig": {
            "faces": []
          }
        },
        "id_str": "2037661906459099137",
        "indices": [
          270,
          293
        ],
        "media_key": "3_2037661906459099137",
        "media_results": {
          "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARxHOrU61wABCgACHEc76b4bYM8AAA==",
          "result": {
            "__typename": "ApiMedia",
            "id": "QXBpTWVkaWE6DAABCgABHEc6tTrXAAEKAAIcRzvpvhtgzwAA",
            "media_key": "3_2037661906459099137"
          }
        },
        "media_url_https": "https://pbs.twimg.com/media/HEc6tTrXAAEtHYU.jpg",
        "original_info": {
          "focus_rects": [
            {
              "h": 833,
              "w": 1488,
              "x": 65,
              "y": 0
            },
            {
              "h": 833,
              "w": 833,
              "x": 720,
              "y": 0
            },
            {
              "h": 833,
              "w": 731,
              "x": 822,
              "y": 0
            },
            {
              "h": 833,
              "w": 417,
              "x": 1136,
              "y": 0
            },
            {
              "h": 833,
              "w": 1553,
              "x": 0,
              "y": 0
            }
          ],
          "height": 833,
          "width": 1553
        },
        "sizes": {
          "large": {
            "h": 833,
            "w": 1553
          }
        },
        "type": "photo",
        "url": "https://t.co/LtEjoVlpMJ"
      }
    ]
  },
  "card": null,
  "place": {},
  "entities": {
    "hashtags": [],
    "symbols": [],
    "urls": [],
    "user_mentions": [
      {
        "id_str": "865622395",
        "indices": [
          345,
          351
        ],
        "name": "Sebastian Raschka",
        "screen_name": "rasbt"
      },
      {
        "id_str": "3290526443",
        "indices": [
          614,
          626
        ],
        "name": "Jia-Bin Huang",
        "screen_name": "jbhuang0604"
      }
    ]
  },
  "quoted_tweet": {
    "type": "tweet",
    "id": "2037297869518950430",
    "url": "https://x.com/levidiamode/status/2037297869518950430",
    "twitterUrl": "https://twitter.com/levidiamode/status/2037297869518950430",
    "text": "Day 82/365 of GPU Programming\n\nTaking a closer look at Mixture of Experts today, so I can write better MoE kernels. Specifically, to optimize an MXFP4 MoE fused kernel for the GPU Mode challenge.\n\nI haven't had much prior exposure to MoEs, so lots of new concepts I learned today. Luckily I found the best intro to MoEs thanks to @MaartenGr visual overview of the topic.\n\nI then watched @tatsu_hashimoto's amazing Stanford CS336 lecture on MoEs, which added deeper context around why MoEs are gaining popularity, FLOPs, OLMoE, infra complexity, routing functions (mindblown this works so well...), expert sizes, training objectives, top k routing and DeepSeek variations. \n\nOnce I had a basic understanding I started playing around with the some AITER kernels but progress there is tbd.\n\nAlso had a nice chat with @juscallmevyom (who was kind enough to reach out!) about the AMD kernels and the challenge of materialization overhead.",
    "source": "Twitter for iPhone",
    "retweetCount": 40,
    "replyCount": 8,
    "likeCount": 517,
    "quoteCount": 2,
    "viewCount": 42413,
    "createdAt": "Thu Mar 26 22:37:17 +0000 2026",
    "lang": "en",
    "bookmarkCount": 477,
    "isReply": false,
    "inReplyToId": null,
    "conversationId": "2037297869518950430",
    "displayTextRange": [
      0,
      280
    ],
    "inReplyToUserId": null,
    "inReplyToUsername": null,
    "author": {
      "type": "user",
      "userName": "levidiamode",
      "url": "https://x.com/levidiamode",
      "twitterUrl": "https://twitter.com/levidiamode",
      "id": "1933932995528044544",
      "name": "levi",
      "isVerified": false,
      "isBlueVerified": true,
      "verifiedType": null,
      "profilePicture": "https://pbs.twimg.com/profile_images/2005322181538983936/KvzCyzRA_normal.jpg",
      "coverPicture": "https://pbs.twimg.com/profile_banners/1933932995528044544/1766941466",
      "description": "",
      "location": "out of memory",
      "followers": 1496,
      "following": 404,
      "status": "",
      "canDm": false,
      "canMediaTag": true,
      "createdAt": "Sat Jun 14 17:02:38 +0000 2025",
      "entities": {
        "description": {
          "urls": []
        },
        "url": {}
      },
      "fastFollowersCount": 0,
      "favouritesCount": 1105,
      "hasCustomTimelines": true,
      "isTranslator": false,
      "mediaCount": 88,
      "statusesCount": 223,
      "withheldInCountries": [],
      "affiliatesHighlightedLabel": {},
      "possiblySensitive": false,
      "pinnedTweetIds": [
        "2034732672845488598"
      ],
      "profile_bio": {
        "description": "365 days of GPU programming \n▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 83/365",
        "entities": {
          "description": {
            "hashtags": [],
            "symbols": [],
            "urls": [],
            "user_mentions": []
          },
          "url": {
            "urls": [
              {
                "display_url": "substack.com/@levidiamode",
                "expanded_url": "https://substack.com/@levidiamode",
                "indices": [
                  0,
                  23
                ],
                "url": "https://t.co/I4wYT0Z3IY"
              }
            ]
          }
        }
      },
      "isAutomated": false,
      "automatedBy": null
    },
    "extendedEntities": {
      "media": [
        {
          "allow_download_status": {
            "allow_download": true
          },
          "display_url": "pic.twitter.com/D0ApTW2lPv",
          "expanded_url": "https://twitter.com/levidiamode/status/2037297869518950430/photo/1",
          "ext_media_availability": {
            "status": "Available"
          },
          "features": {
            "large": {
              "faces": [
                {
                  "h": 159,
                  "w": 159,
                  "x": 1797,
                  "y": 915
                },
                {
                  "h": 225,
                  "w": 225,
                  "x": 895,
                  "y": 783
                }
              ]
            },
            "orig": {
              "faces": [
                {
                  "h": 196,
                  "w": 196,
                  "x": 2208,
                  "y": 1125
                },
                {
                  "h": 277,
                  "w": 277,
                  "x": 1100,
                  "y": 963
                }
              ]
            }
          },
          "id_str": "2037275153802121216",
          "indices": [
            281,
            304
          ],
          "media_key": "3_2037275153802121216",
          "media_results": {
            "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARxF2vVcGtAACgACHEXvnkZWUB4AAA==",
            "result": {
              "__typename": "ApiMedia",
              "id": "QXBpTWVkaWE6DAABCgABHEXa9Vwa0AAKAAIcRe+eRlZQHgAA",
              "media_key": "3_2037275153802121216"
            }
          },
          "media_url_https": "https://pbs.twimg.com/media/HEXa9Vwa0AAb-CG.jpg",
          "original_info": {
            "focus_rects": [
              {
                "h": 1409,
                "w": 2516,
                "x": 0,
                "y": 18
              },
              {
                "h": 1427,
                "w": 1427,
                "x": 607,
                "y": 0
              },
              {
                "h": 1427,
                "w": 1252,
                "x": 694,
                "y": 0
              },
              {
                "h": 1427,
                "w": 714,
                "x": 963,
                "y": 0
              },
              {
                "h": 1427,
                "w": 2516,
                "x": 0,
                "y": 0
              }
            ],
            "height": 1427,
            "width": 2516
          },
          "sizes": {
            "large": {
              "h": 1162,
              "w": 2048
            }
          },
          "type": "photo",
          "url": "https://t.co/D0ApTW2lPv"
        },
        {
          "allow_download_status": {
            "allow_download": true
          },
          "display_url": "pic.twitter.com/D0ApTW2lPv",
          "expanded_url": "https://twitter.com/levidiamode/status/2037297869518950430/photo/1",
          "ext_media_availability": {
            "status": "Available"
          },
          "features": {
            "large": {
              "faces": []
            },
            "orig": {
              "faces": []
            }
          },
          "id_str": "2037294957636329472",
          "indices": [
            281,
            304
          ],
          "media_key": "3_2037294957636329472",
          "media_results": {
            "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARxF7PhMm6AACgACHEXvnkZWUB4AAA==",
            "result": {
              "__typename": "ApiMedia",
              "id": "QXBpTWVkaWE6DAABCgABHEXs+EyboAAKAAIcRe+eRlZQHgAA",
              "media_key": "3_2037294957636329472"
            }
          },
          "media_url_https": "https://pbs.twimg.com/media/HEXs-EyboAATOAx.jpg",
          "original_info": {
            "focus_rects": [
              {
                "h": 927,
                "w": 1655,
                "x": 111,
                "y": 0
              },
              {
                "h": 927,
                "w": 927,
                "x": 817,
                "y": 0
              },
              {
                "h": 927,
                "w": 813,
                "x": 874,
                "y": 0
              },
              {
                "h": 927,
                "w": 464,
                "x": 1048,
                "y": 0
              },
              {
                "h": 927,
                "w": 1766,
                "x": 0,
                "y": 0
              }
            ],
            "height": 927,
            "width": 1766
          },
          "sizes": {
            "large": {
              "h": 927,
              "w": 1766
            }
          },
          "type": "photo",
          "url": "https://t.co/D0ApTW2lPv"
        },
        {
          "allow_download_status": {
            "allow_download": true
          },
          "display_url": "pic.twitter.com/D0ApTW2lPv",
          "expanded_url": "https://twitter.com/levidiamode/status/2037297869518950430/photo/1",
          "ext_media_availability": {
            "status": "Available"
          },
          "features": {
            "large": {
              "faces": [
                {
                  "h": 207,
                  "w": 207,
                  "x": 658,
                  "y": 191
                }
              ]
            },
            "orig": {
              "faces": [
                {
                  "h": 207,
                  "w": 207,
                  "x": 658,
                  "y": 191
                }
              ]
            }
          },
          "id_str": "2037296284780908545",
          "indices": [
            281,
            304
          ],
          "media_key": "3_2037296284780908545",
          "media_results": {
            "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARxF7i1MltABCgACHEXvnkZWUB4AAA==",
            "result": {
              "__typename": "ApiMedia",
              "id": "QXBpTWVkaWE6DAABCgABHEXuLUyW0AEKAAIcRe+eRlZQHgAA",
              "media_key": "3_2037296284780908545"
            }
          },
          "media_url_https": "https://pbs.twimg.com/media/HEXuLUyW0AEDdgj.jpg",
          "original_info": {
            "focus_rects": [
              {
                "h": 921,
                "w": 1645,
                "x": 0,
                "y": 35
              },
              {
                "h": 956,
                "w": 956,
                "x": 0,
                "y": 0
              },
              {
                "h": 956,
                "w": 839,
                "x": 0,
                "y": 0
              },
              {
                "h": 956,
                "w": 478,
                "x": 131,
                "y": 0
              },
              {
                "h": 956,
                "w": 1645,
                "x": 0,
                "y": 0
              }
            ],
            "height": 956,
            "width": 1645
          },
          "sizes": {
            "large": {
              "h": 956,
              "w": 1645
            }
          },
          "type": "photo",
          "url": "https://t.co/D0ApTW2lPv"
        },
        {
          "allow_download_status": {
            "allow_download": true
          },
          "display_url": "pic.twitter.com/D0ApTW2lPv",
          "expanded_url": "https://twitter.com/levidiamode/status/2037297869518950430/photo/1",
          "ext_media_availability": {
            "status": "Available"
          },
          "features": {
            "large": {
              "faces": [
                {
                  "h": 179,
                  "w": 179,
                  "x": 1549,
                  "y": 303
                },
                {
                  "h": 189,
                  "w": 189,
                  "x": 1095,
                  "y": 297
                }
              ]
            },
            "orig": {
              "faces": [
                {
                  "h": 180,
                  "w": 180,
                  "x": 1557,
                  "y": 305
                },
                {
                  "h": 190,
                  "w": 190,
                  "x": 1101,
                  "y": 299
                }
              ]
            }
          },
          "id_str": "2037296520777662464",
          "indices": [
            281,
            304
          ],
          "media_key": "3_2037296520777662464",
          "media_results": {
            "id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARxF7mQ/F4AACgACHEXvnkZWUB4AAA==",
            "result": {
              "__typename": "ApiMedia",
              "id": "QXBpTWVkaWE6DAABCgABHEXuZD8XgAAKAAIcRe+eRlZQHgAA",
              "media_key": "3_2037296520777662464"
            }
          },
          "media_url_https": "https://pbs.twimg.com/media/HEXuZD8XgAAVkZz.jpg",
          "original_info": {
            "focus_rects": [
              {
                "h": 1120,
                "w": 2000,
                "x": 0,
                "y": 0
              },
              {
                "h": 1120,
                "w": 1120,
                "x": 0,
                "y": 0
              },
              {
                "h": 1120,
                "w": 982,
                "x": 0,
                "y": 0
              },
              {
                "h": 1120,
                "w": 560,
                "x": 0,
                "y": 0
              },
              {
                "h": 1120,
                "w": 2058,
                "x": 0,
                "y": 0
              }
            ],
            "height": 1120,
            "width": 2058
          },
          "sizes": {
            "large": {
              "h": 1115,
              "w": 2048
            }
          },
          "type": "photo",
          "url": "https://t.co/D0ApTW2lPv"
        }
      ]
    },
    "card": null,
    "place": {},
    "entities": {
      "hashtags": [],
      "symbols": [],
      "urls": [],
      "user_mentions": [
        {
          "id_str": "1260142508748804096",
          "indices": [
            330,
            340
          ],
          "name": "Maarten Grootendorst",
          "screen_name": "MaartenGr"
        },
        {
          "id_str": "1118495199863476225",
          "indices": [
            387,
            403
          ],
          "name": "Tatsunori Hashimoto",
          "screen_name": "tatsu_hashimoto"
        },
        {
          "id_str": "1102252093711233025",
          "indices": [
            814,
            828
          ],
          "name": "vyomakesh",
          "screen_name": "juscallmevyom"
        }
      ]
    },
    "quoted_tweet": {
      "type": "tweet",
      "id": "2036946636715241788",
      "url": "",
      "twitterUrl": "",
      "text": "",
      "source": "Twitter for iPhone",
      "retweetCount": 0,
      "replyCount": 0,
      "likeCount": 0,
      "quoteCount": 0,
      "viewCount": 0,
      "createdAt": "",
      "lang": "",
      "bookmarkCount": 0,
      "isReply": false,
      "inReplyToId": null,
      "conversationId": "",
      "displayTextRange": [],
      "inReplyToUserId": null,
      "inReplyToUsername": null,
      "author": {},
      "extendedEntities": {},
      "card": null,
      "place": {},
      "entities": {},
      "quoted_tweet": null,
      "retweeted_tweet": null,
      "isLimitedReply": false,
      "article": null
    },
    "retweeted_tweet": null,
    "isLimitedReply": false,
    "article": null
  },
  "retweeted_tweet": null,
  "isLimitedReply": false,
  "article": null
}