Viewing enriched Twitter post
So it turns out that using LoRAs to customize a general LLM (the way Apple tunes its on-device models) limit the LLM far more than fine-tuning, because they lose some of their ability to generalize. The reason is LoRA's add ominously-labelled intruder dimensions.
{
"media": [
{
"url": "https://crmoxkoizveukayfjuyo.supabase.co/storage/v1/object/public/media/posts/1854648003715551641/media_0.jpg",
"type": "photo",
"original_url": "https://pbs.twimg.com/media/Gb0IGSHbwAEeMSX.jpg",
"download_date": "2025-08-13T05:51:54.575326",
"stored_in_supabase": true,
"format_converted_from_list": true
}
],
"conversion_date": "2025-08-13T00:32:28.038018",
"format_converted": true,
"original_structure": "had_media_only"
}{
"user": {
"created_at": "2009-05-10T22:33:52.000Z",
"default_profile_image": false,
"description": "Professor @Wharton studying AI, innovation & startups. Democratizing education using tech\nBook: https://t.co/CSmipbJ2jV\nSubstack: https://t.co/UIBhxu4bgq",
"fast_followers_count": 0,
"favourites_count": 6584,
"followers_count": 242516,
"friends_count": 557,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 5555,
"location": "Philadelphia, PA",
"media_count": 12485,
"name": "Ethan Mollick",
"normal_followers_count": 242516,
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/39125788/1713840725",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1601382188712398850/3AAOlqrX_normal.jpg",
"screen_name": "emollick",
"statuses_count": 29379,
"translator_type": "none",
"url": "https://t.co/uItckI9ujc",
"verified": true,
"withheld_in_countries": [],
"id_str": "39125788"
},
"id": "1854648003715551641",
"conversation_id": "1854648003715551641",
"full_text": "So it turns out that using LoRAs to customize a general LLM (the way Apple tunes its on-device models) limit the LLM far more than fine-tuning, because they lose some of their ability to generalize.\n\nThe reason is LoRA's add ominously-labelled intruder dimensions.",
"reply_count": 9,
"retweet_count": 21,
"favorite_count": 230,
"hashtags": [],
"symbols": [],
"user_mentions": [],
"urls": [],
"media": [
{
"media_url": "https://pbs.twimg.com/media/Gb0IGSHbwAEeMSX.jpg",
"type": "photo"
}
],
"url": "https://twitter.com/emollick/status/1854648003715551641",
"created_at": "2024-11-07T22:11:56.000Z",
"#sort_index": "1854648003715551641",
"view_count": 32720,
"quote_count": 1,
"is_quote_tweet": true,
"is_retweet": false,
"is_pinned": false,
"is_truncated": false,
"quoted_tweet": {
"user": {
"created_at": "2020-10-22T21:56:32.000Z",
"default_profile_image": false,
"description": "ML engineer & speculator • extremely silly jester",
"fast_followers_count": 0,
"favourites_count": 56736,
"followers_count": 3452,
"friends_count": 1447,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 43,
"location": "",
"media_count": 2028,
"name": "kalomaze",
"normal_followers_count": 3452,
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1319397157913436163/1691885214",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1866973763448803328/J7gILxEb_normal.jpg",
"screen_name": "kalomaze",
"statuses_count": 11573,
"translator_type": "none",
"verified": true,
"withheld_in_countries": [],
"id_str": "1319397157913436163"
},
"id": "1854603984608038983",
"conversation_id": "1854603984608038983",
"full_text": "https://t.co/l0XSIgjnNe",
"reply_count": 23,
"retweet_count": 81,
"favorite_count": 673,
"hashtags": [],
"symbols": [],
"user_mentions": [],
"urls": [
{
"url": "https://t.co/l0XSIgjnNe",
"expanded_url": "https://arxiv.org/abs/2410.21228v1",
"display_url": "arxiv.org/abs/2410.21228…"
}
],
"media": [],
"url": "https://twitter.com/kalomaze/status/1854603984608038983",
"created_at": "2024-11-07T19:17:01.000Z",
"#sort_index": "1854648003715551700",
"view_count": 276611,
"quote_count": 17,
"is_quote_tweet": false,
"is_retweet": false,
"is_pinned": false,
"is_truncated": false
},
"startUrl": "https://x.com/emollick/status/1854648003715551641"
}