Viewing enriched Twitter post
How often do LLMs claim to prove false mathematical statements? In our latest benchmark, BrokenArXiv, we find they do so very often. The best model, GPT-5.4, only rejects 40% of incorrect statements obtained by perturbing recent ArXiv papers, and other models do much worse. https://t.co/RRQNZfnCtW
{
"media": [
{
"url": "https://crmoxkoizveukayfjuyo.supabase.co/storage/v1/object/public/media/posts/2032458037823483953/media_0.jpg",
"media_url": "https://crmoxkoizveukayfjuyo.supabase.co/storage/v1/object/public/media/posts/2032458037823483953/media_0.jpg",
"type": "photo",
"filename": "media_0.jpg"
}
],
"processed_at": "2026-03-13T17:39:21.332209",
"pipeline_version": "2.0"
}{
"type": "tweet",
"id": "2032458037823483953",
"url": "https://x.com/j_dekoninck/status/2032458037823483953",
"twitterUrl": "https://twitter.com/j_dekoninck/status/2032458037823483953",
"text": "How often do LLMs claim to prove false mathematical statements?\n\nIn our latest benchmark, BrokenArXiv, we find they do so very often. The best model, GPT-5.4, only rejects 40% of incorrect statements obtained by perturbing recent ArXiv papers, and other models do much worse. https://t.co/RRQNZfnCtW",
"source": "Twitter for iPhone",
"retweetCount": 44,
"replyCount": 9,
"likeCount": 302,
"quoteCount": 5,
"viewCount": 18581,
"createdAt": "Fri Mar 13 14:05:32 +0000 2026",
"lang": "en",
"bookmarkCount": 87,
"isReply": false,
"inReplyToId": null,
"conversationId": "2032458037823483953",
"displayTextRange": [
0,
275
],
"inReplyToUserId": null,
"inReplyToUsername": null,
"author": {
"type": "user",
"userName": "j_dekoninck",
"url": "https://x.com/j_dekoninck",
"twitterUrl": "https://twitter.com/j_dekoninck",
"id": "1664632458703261700",
"name": "Jasper Dekoninck",
"isVerified": false,
"isBlueVerified": false,
"verifiedType": null,
"profilePicture": "https://pbs.twimg.com/profile_images/1889075959472476162/D9M1omhy_normal.jpg",
"coverPicture": "",
"description": "",
"location": "",
"followers": 2410,
"following": 127,
"status": "",
"canDm": false,
"canMediaTag": true,
"createdAt": "Fri Jun 02 13:58:10 +0000 2023",
"entities": {
"description": {
"urls": []
},
"url": {}
},
"fastFollowersCount": 0,
"favouritesCount": 268,
"hasCustomTimelines": true,
"isTranslator": false,
"mediaCount": 123,
"statusesCount": 576,
"withheldInCountries": [],
"affiliatesHighlightedLabel": {},
"possiblySensitive": false,
"pinnedTweetIds": [],
"profile_bio": {
"description": "PhD Student at ETH Zurich, working on mathematical evaluation of LLMs (MathArena, IMproofBench, BrokenMath, ...)",
"entities": {
"description": {
"hashtags": [],
"symbols": [],
"urls": [],
"user_mentions": []
}
}
},
"isAutomated": false,
"automatedBy": null
},
"extendedEntities": {
"media": [
{
"display_url": "pic.twitter.com/RRQNZfnCtW",
"expanded_url": "https://twitter.com/j_dekoninck/status/2032458037823483953/photo/1",
"ext_media_availability": {
"status": "Available"
},
"features": {
"large": {
"faces": []
},
"orig": {
"faces": []
}
},
"id_str": "2032458034862239745",
"indices": [
276,
299
],
"media_key": "3_2032458034862239745",
"media_results": {
"id": "QXBpTWVkaWFSZXN1bHRzOgwAAQoAARw0vdB02jABCgACHDS90SVbMDEAAA==",
"result": {
"__typename": "ApiMedia",
"id": "QXBpTWVkaWE6DAABCgABHDS90HTaMAEKAAIcNL3RJVswMQAA",
"media_key": "3_2032458034862239745"
}
},
"media_url_https": "https://pbs.twimg.com/media/HDS90HTaMAExfRI.jpg",
"original_info": {
"focus_rects": [
{
"h": 935,
"w": 1670,
"x": 287,
"y": 0
},
{
"h": 935,
"w": 935,
"x": 655,
"y": 0
},
{
"h": 935,
"w": 820,
"x": 712,
"y": 0
},
{
"h": 935,
"w": 468,
"x": 888,
"y": 0
},
{
"h": 935,
"w": 2993,
"x": 0,
"y": 0
}
],
"height": 935,
"width": 2993
},
"sizes": {
"large": {
"h": 640,
"w": 2048
}
},
"type": "photo",
"url": "https://t.co/RRQNZfnCtW"
}
]
},
"card": null,
"place": {},
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [],
"user_mentions": []
},
"quoted_tweet": null,
"retweeted_tweet": null,
"isLimitedReply": false,
"article": null
}