{
  "type": "tweet",
  "id": "2062890225144135800",
  "url": "https://x.com/steverab/status/2062890225144135800",
  "twitterUrl": "https://twitter.com/steverab/status/2062890225144135800",
  "text": "Very excited to share that our paper \"Towards a Science of AI Agent Reliability\" was accepted at ICML 2026! See you in Seoul! 🎉\n\nWe just released our camera ready version with three important updates (details below). We also recorded a short video on the paper's contributions.\n\nMain changes (full discussion at https://t.co/1a5r1jNFF4):\n1️⃣We have added the latest set of frontier models to our evaluation (GPT 5.5, Gemini 3.1 Pro and 3.5 Flash, and Claude Opus 4.7) and find that they are not meaningfully more reliable than previously released models. Agent reliability is still far from being solved.\n2️⃣We have updated the definition and measurement of our outcome consistency metric, which contained a typo in the pre-print we initially released. This caused us to under-estimate outcome consistency in our initial set of results. We have updated the paper and our codebase to the corrected metric. Despite this change, our new results show that outcome consistency is still surprisingly low across many reported models.\n3️⃣We discovered multiple issues in our HAL Generalist Agent scaffold that we used for our experiments on GAIA. Notably, we discovered multiple instances of answer leakage and agents cheating on our evaluation. This caused us to slightly over-estimate both accuracy and reliability. At the same time, we noticed that the scaffold was overly constrained in terms of permissible software library imports. This caused us to slightly under-estimate both accuracy and reliability. We have done a rigorous audit of the scaffold and have fixed those issues. Overall, we saw that our resulting accuracy and reliability numbers are not meaningfully impacted by this change when compared to our original numbers.\n\n📄Our paper: https://t.co/HAKHzASrOZ\n📊Our dashboard: https://t.co/apbtxtsdvz\n🎥Short video: https://t.co/uqIourw6C6\n\nJoint work w/ @sayashk, @PKirgis, @khl53182440, @SaitejaUtpala, and @random_walker.",
  "source": "Twitter for iPhone",
  "retweetCount": 36,
  "replyCount": 13,
  "likeCount": 249,
  "quoteCount": 4,
  "viewCount": 23872,
  "createdAt": "Fri Jun 05 13:32:10 +0000 2026",
  "lang": "en",
  "bookmarkCount": 255,
  "isReply": false,
  "inReplyToId": null,
  "conversationId": "2062890225144135800",
  "displayTextRange": [
    0,
    277
  ],
  "inReplyToUserId": null,
  "inReplyToUsername": null,
  "author": {
    "type": "user",
    "userName": "steverab",
    "url": "https://x.com/steverab",
    "twitterUrl": "https://twitter.com/steverab",
    "id": "138821636",
    "name": "Stephan Rabanser",
    "isVerified": false,
    "isBlueVerified": false,
    "verifiedType": null,
    "profilePicture": "https://pbs.twimg.com/profile_images/1928214170547159040/VekssmRX_normal.jpg",
    "coverPicture": "https://pbs.twimg.com/profile_banners/138821636/1760330288",
    "description": "",
    "location": "Princeton, NJ",
    "followers": 787,
    "following": 384,
    "status": "",
    "canDm": true,
    "canMediaTag": false,
    "createdAt": "Fri Apr 30 18:05:23 +0000 2010",
    "entities": {
      "description": {
        "urls": []
      },
      "url": {}
    },
    "fastFollowersCount": 0,
    "favouritesCount": 489,
    "hasCustomTimelines": true,
    "isTranslator": false,
    "mediaCount": 1049,
    "statusesCount": 10125,
    "withheldInCountries": [],
    "affiliatesHighlightedLabel": {},
    "possiblySensitive": false,
    "pinnedTweetIds": [
      "2062890225144135800"
    ],
    "profile_bio": {
      "description": "Postdoctoral Researcher @Princeton. Reliable, safe, trustworthy machine learning. Previously: @UofT @VectorInst @TU_Muenchen @Google @awscloud",
      "entities": {
        "description": {
          "user_mentions": [
            {
              "id_str": "",
              "indices": [
                24,
                34
              ],
              "name": "",
              "screen_name": "Princeton"
            },
            {
              "id_str": "",
              "indices": [
                94,
                99
              ],
              "name": "",
              "screen_name": "UofT"
            },
            {
              "id_str": "",
              "indices": [
                100,
                111
              ],
              "name": "",
              "screen_name": "VectorInst"
            },
            {
              "id_str": "",
              "indices": [
                112,
                124
              ],
              "name": "",
              "screen_name": "TU_Muenchen"
            },
            {
              "id_str": "",
              "indices": [
                125,
                132
              ],
              "name": "",
              "screen_name": "Google"
            },
            {
              "id_str": "",
              "indices": [
                133,
                142
              ],
              "name": "",
              "screen_name": "awscloud"
            }
          ]
        },
        "url": {
          "urls": [
            {
              "display_url": "rabanser.dev",
              "expanded_url": "https://rabanser.dev",
              "indices": [
                0,
                23
              ],
              "url": "https://t.co/cNNItOKNEM"
            }
          ]
        }
      }
    },
    "isAutomated": false,
    "automatedBy": null
  },
  "extendedEntities": {
    "media": [
      {
        "additional_media_info": {
          "monetizable": true
        },
        "allow_download_status": {
          "allow_download": true
        },
        "display_url": "pic.twitter.com/6BNxJfhheK",
        "expanded_url": "https://twitter.com/steverab/status/2062890225144135800/video/1",
        "ext_master_playlist_only": [],
        "ext_media_availability": {
          "status": "Available"
        },
        "ext_playlists": [],
        "id_str": "2062012973464322048",
        "indices": [
          278,
          301
        ],
        "media_key": "13_2062012973464322048",
        "media_results": {
          "id": "QXBpTWVkaWFSZXN1bHRzOgwABAoAARydvd/VloAAAAA=",
          "result": {
            "__typename": "ApiMedia",
            "id": "QXBpTWVkaWE6DAAECgABHJ2939WWgAAAAA==",
            "media_key": "13_2062012973464322048"
          }
        },
        "media_url_https": "https://pbs.twimg.com/amplify_video_thumb/2062012973464322048/img/KhCtBvvuugpPfvsI.jpg",
        "original_info": {
          "focus_rects": [],
          "height": 1440,
          "width": 2560
        },
        "sizes": {
          "large": {
            "h": 1152,
            "w": 2048
          }
        },
        "type": "video",
        "url": "https://t.co/6BNxJfhheK",
        "video_info": {
          "aspect_ratio": [
            16,
            9
          ],
          "duration_millis": 321170,
          "variants": [
            {
              "content_type": "application/x-mpegURL",
              "url": "https://video.twimg.com/amplify_video/2062012973464322048/pl/MkqfUVtmMJJuzdYu.m3u8?tag=27&v=fa9"
            },
            {
              "bitrate": 256000,
              "content_type": "video/mp4",
              "url": "https://video.twimg.com/amplify_video/2062012973464322048/vid/avc1/480x270/GkgY2q07dgiiCX5H.mp4?tag=27"
            },
            {
              "bitrate": 832000,
              "content_type": "video/mp4",
              "url": "https://video.twimg.com/amplify_video/2062012973464322048/vid/avc1/640x360/5r4alJHVQDetl9n9.mp4?tag=27"
            },
            {
              "bitrate": 2176000,
              "content_type": "video/mp4",
              "url": "https://video.twimg.com/amplify_video/2062012973464322048/vid/avc1/1280x720/qwqZPfvoXts101Na.mp4?tag=27"
            },
            {
              "bitrate": 10368000,
              "content_type": "video/mp4",
              "url": "https://video.twimg.com/amplify_video/2062012973464322048/vid/avc1/2560x1440/y8XOes76bzRFLI8E.mp4?tag=27"
            }
          ]
        }
      }
    ]
  },
  "card": null,
  "place": {},
  "entities": {
    "hashtags": [],
    "symbols": [],
    "timestamps": [],
    "urls": [
      {
        "display_url": "hal.cs.princeton.edu/reliability/#u…",
        "expanded_url": "https://hal.cs.princeton.edu/reliability/#updates",
        "indices": [
          312,
          335
        ],
        "url": "https://t.co/1a5r1jNFF4"
      },
      {
        "display_url": "arxiv.org/abs/2602.16666",
        "expanded_url": "https://arxiv.org/abs/2602.16666",
        "indices": [
          1743,
          1766
        ],
        "url": "https://t.co/HAKHzASrOZ"
      },
      {
        "display_url": "hal.cs.princeton.edu/reliability/",
        "expanded_url": "https://hal.cs.princeton.edu/reliability/",
        "indices": [
          1783,
          1806
        ],
        "url": "https://t.co/apbtxtsdvz"
      },
      {
        "display_url": "youtu.be/qftDfEft7U0",
        "expanded_url": "https://youtu.be/qftDfEft7U0",
        "indices": [
          1821,
          1844
        ],
        "url": "https://t.co/uqIourw6C6"
      }
    ],
    "user_mentions": [
      {
        "id_str": "3084274082",
        "indices": [
          1860,
          1868
        ],
        "name": "Sayash Kapoor",
        "screen_name": "sayashk"
      },
      {
        "id_str": "1036066345547444225",
        "indices": [
          1870,
          1878
        ],
        "name": "Peter Kirgis",
        "screen_name": "PKirgis"
      },
      {
        "id_str": "1583796980463325186",
        "indices": [
          1880,
          1892
        ],
        "name": "Kangheng Liu",
        "screen_name": "khl53182440"
      },
      {
        "id_str": "1301938969622376448",
        "indices": [
          1894,
          1908
        ],
        "name": "Saiteja Utpala",
        "screen_name": "SaitejaUtpala"
      },
      {
        "id_str": "10834752",
        "indices": [
          1914,
          1928
        ],
        "name": "Arvind Narayanan",
        "screen_name": "random_walker"
      }
    ]
  },
  "quoted_tweet": null,
  "retweeted_tweet": null,
  "isLimitedReply": false,
  "communityInfo": null,
  "article": null
}