@ivanleomk
Full article is here at https://t.co/WubWjkccQe with the full code available too I tried to make it easy to follow along with most things being derived from scratch with everything I learnt about the BPE tokenizer in the last 2 weekends!
🐦 Twitter Post Details
Viewing enriched Twitter post
📊 Media Metadata
{
"media": [
{
"type": "photo",
"url": "https://crmoxkoizveukayfjuyo.supabase.co/storage/v1/object/public/media/posts/1997297697997697125/media_0.jpg?",
"filename": "media_0.jpg"
}
],
"processed_at": "2025-12-08T13:34:38.530337",
"pipeline_version": "2.0"
}🔧 Raw API Response
{
"type": "tweet",
"id": "1997297697997697125",
"url": "https://x.com/ivanleomk/status/1997297697997697125",
"twitterUrl": "https://twitter.com/ivanleomk/status/1997297697997697125",
"text": "Full article is here at https://t.co/WubWjkccQe with the full code available too\n\nI tried to make it easy to follow along with most things being derived from scratch with everything I learnt about the BPE tokenizer in the last 2 weekends!",
"source": "Twitter for iPhone",
"retweetCount": 0,
"replyCount": 0,
"likeCount": 5,
"quoteCount": 0,
"viewCount": 138,
"createdAt": "Sat Dec 06 13:30:53 +0000 2025",
"lang": "en",
"bookmarkCount": 2,
"isReply": true,
"inReplyToId": "1997297695963382069",
"conversationId": "1997297695963382069",
"displayTextRange": [
0,
238
],
"inReplyToUserId": null,
"inReplyToUsername": null,
"author": {
"type": "user",
"userName": "ivanleomk",
"url": "https://x.com/ivanleomk",
"twitterUrl": "https://twitter.com/ivanleomk",
"id": "267160799",
"name": "Ivan Leo",
"isVerified": false,
"isBlueVerified": true,
"verifiedType": null,
"profilePicture": "https://pbs.twimg.com/profile_images/1838778744468836353/utYfioiO_normal.jpg",
"coverPicture": "https://pbs.twimg.com/profile_banners/267160799/1716692627",
"description": "i work on agents at manus and write at https://t.co/02byZNKMa8.",
"location": "Singapore",
"followers": 4312,
"following": 1398,
"status": "",
"canDm": true,
"canMediaTag": false,
"createdAt": "Wed Mar 16 12:43:36 +0000 2011",
"entities": {
"description": {
"urls": [
{
"display_url": "ivanleo.com",
"expanded_url": "http://ivanleo.com",
"url": "https://t.co/02byZNKMa8",
"indices": [
39,
62
]
}
]
},
"url": {
"urls": [
{
"display_url": "ivanleo.com",
"expanded_url": "http://ivanleo.com",
"url": "https://t.co/02byZNKMa8",
"indices": [
0,
23
]
}
]
}
},
"fastFollowersCount": 0,
"favouritesCount": 4855,
"hasCustomTimelines": true,
"isTranslator": false,
"mediaCount": 1099,
"statusesCount": 6355,
"withheldInCountries": [],
"affiliatesHighlightedLabel": {
"label": {
"url": {
"url": "https://twitter.com/ManusAI",
"urlType": "DeepLink"
},
"badge": {
"url": "https://pbs.twimg.com/profile_images/1972973045393592322/e11bi-fE_bigger.jpg"
},
"description": "Manus",
"userLabelType": "BusinessLabel",
"userLabelDisplayType": "Badge"
}
},
"possiblySensitive": false,
"pinnedTweetIds": [
"1875777259346509940"
],
"profile_bio": {},
"isAutomated": false,
"automatedBy": null
},
"extendedEntities": {},
"card": {
"binding_values": [
{
"key": "photo_image_full_size_large",
"value": {
"image_value": {
"height": 419,
"width": 800,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=800x419"
},
"type": "IMAGE"
}
},
{
"key": "thumbnail_image",
"value": {
"image_value": {
"height": 147,
"width": 280,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=280x150"
},
"type": "IMAGE"
}
},
{
"key": "description",
"value": {
"string_value": "Building a Byte-Pair Encoding tokenizer from scratch",
"type": "STRING"
}
},
{
"key": "domain",
"value": {
"string_value": "ivanleo.com",
"type": "STRING"
}
},
{
"key": "thumbnail_image_large",
"value": {
"image_value": {
"height": 315,
"width": 600,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=600x600"
},
"type": "IMAGE"
}
},
{
"key": "summary_photo_image_small",
"value": {
"image_value": {
"height": 202,
"width": 386,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=386x202"
},
"type": "IMAGE"
}
},
{
"key": "thumbnail_image_original",
"value": {
"image_value": {
"height": 630,
"width": 1200,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=orig"
},
"type": "IMAGE"
}
},
{
"key": "photo_image_full_size_small",
"value": {
"image_value": {
"height": 202,
"width": 386,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=386x202"
},
"type": "IMAGE"
}
},
{
"key": "summary_photo_image_large",
"value": {
"image_value": {
"height": 419,
"width": 800,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=800x419"
},
"type": "IMAGE"
}
},
{
"key": "thumbnail_image_small",
"value": {
"image_value": {
"height": 76,
"width": 144,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=144x144"
},
"type": "IMAGE"
}
},
{
"key": "thumbnail_image_x_large",
"value": {
"image_value": {
"height": 630,
"width": 1200,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=2048x2048_2_exp"
},
"type": "IMAGE"
}
},
{
"key": "photo_image_full_size_original",
"value": {
"image_value": {
"height": 630,
"width": 1200,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=orig"
},
"type": "IMAGE"
}
},
{
"key": "photo_image_full_size_alt_text",
"value": {
"string_value": "Implementing BPE",
"type": "STRING"
}
},
{
"key": "vanity_url",
"value": {
"scribe_key": "vanity_url",
"string_value": "ivanleo.com",
"type": "STRING"
}
},
{
"key": "photo_image_full_size",
"value": {
"image_value": {
"height": 314,
"width": 600,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=600x314"
},
"type": "IMAGE"
}
},
{
"key": "summary_photo_image_alt_text",
"value": {
"string_value": "Implementing BPE",
"type": "STRING"
}
},
{
"key": "thumbnail_image_color",
"value": {
"image_color_value": {
"palette": [
{
"rgb": {
"blue": 255,
"green": 255,
"red": 255
},
"percentage": 98.8
},
{
"rgb": {
"blue": 126,
"green": 126,
"red": 126
},
"percentage": 1.2
}
]
},
"type": "IMAGE_COLOR"
}
},
{
"key": "title",
"value": {
"string_value": "Implementing BPE",
"type": "STRING"
}
},
{
"key": "summary_photo_image_color",
"value": {
"image_color_value": {
"palette": [
{
"rgb": {
"blue": 255,
"green": 255,
"red": 255
},
"percentage": 98.8
},
{
"rgb": {
"blue": 126,
"green": 126,
"red": 126
},
"percentage": 1.2
}
]
},
"type": "IMAGE_COLOR"
}
},
{
"key": "summary_photo_image_x_large",
"value": {
"image_value": {
"height": 630,
"width": 1200,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=2048x2048_2_exp"
},
"type": "IMAGE"
}
},
{
"key": "summary_photo_image",
"value": {
"image_value": {
"height": 314,
"width": 600,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=600x314"
},
"type": "IMAGE"
}
},
{
"key": "photo_image_full_size_color",
"value": {
"image_color_value": {
"palette": [
{
"rgb": {
"blue": 255,
"green": 255,
"red": 255
},
"percentage": 98.8
},
{
"rgb": {
"blue": 126,
"green": 126,
"red": 126
},
"percentage": 1.2
}
]
},
"type": "IMAGE_COLOR"
}
},
{
"key": "photo_image_full_size_x_large",
"value": {
"image_value": {
"height": 630,
"width": 1200,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=2048x2048_2_exp"
},
"type": "IMAGE"
}
},
{
"key": "card_url",
"value": {
"scribe_key": "card_url",
"string_value": "https://t.co/WubWjkccQe",
"type": "STRING"
}
},
{
"key": "summary_photo_image_original",
"value": {
"image_value": {
"height": 630,
"width": 1200,
"url": "https://pbs.twimg.com/card_img/1997297028930678785/GHdBXlZC?format=png&name=orig"
},
"type": "IMAGE"
}
}
],
"card_platform": {
"platform": {
"audience": {
"name": "production"
},
"device": {
"name": "Android",
"version": "12"
}
}
},
"name": "summary_large_image",
"url": "https://t.co/WubWjkccQe",
"user_refs_results": []
},
"place": {},
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [
{
"display_url": "ivanleo.com/blog/implement…",
"expanded_url": "https://ivanleo.com/blog/implementing-bpe",
"url": "https://t.co/WubWjkccQe",
"indices": [
24,
47
]
}
],
"user_mentions": []
},
"quoted_tweet": null,
"retweeted_tweet": null,
"article": null
}