{
  "type": "tweet",
  "id": "2071031563324662252",
  "url": "https://x.com/GaryMarcus/status/2071031563324662252",
  "twitterUrl": "https://twitter.com/GaryMarcus/status/2071031563324662252",
  "text": "RT @EricTopol: Thanks for running our open-source work on current frontier models\n\n“The results are: the most capable models today (GPT-5.5…",
  "source": "Twitter for iPhone",
  "retweetCount": 42,
  "replyCount": 14,
  "likeCount": 285,
  "quoteCount": 4,
  "viewCount": 58769,
  "createdAt": "Sun Jun 28 00:42:57 +0000 2026",
  "lang": "en",
  "bookmarkCount": 90,
  "isReply": false,
  "inReplyToId": null,
  "conversationId": "2071031563324662252",
  "displayTextRange": [
    0,
    140
  ],
  "inReplyToUserId": null,
  "inReplyToUsername": null,
  "author": {
    "type": "user",
    "userName": "GaryMarcus",
    "url": "https://x.com/GaryMarcus",
    "twitterUrl": "https://twitter.com/GaryMarcus",
    "id": "232294292",
    "name": "Gary Marcus",
    "isVerified": false,
    "isBlueVerified": true,
    "verifiedType": null,
    "profilePicture": "https://pbs.twimg.com/profile_images/2048405471900606464/kPeRHI2z_normal.jpg",
    "coverPicture": "https://pbs.twimg.com/profile_banners/232294292/1727132331",
    "description": "",
    "location": "",
    "followers": 228370,
    "following": 7057,
    "status": "",
    "canDm": true,
    "canMediaTag": false,
    "createdAt": "Thu Dec 30 19:20:12 +0000 2010",
    "entities": {
      "description": {
        "urls": []
      },
      "url": {}
    },
    "fastFollowersCount": 0,
    "favouritesCount": 89487,
    "hasCustomTimelines": true,
    "isTranslator": false,
    "mediaCount": 3835,
    "statusesCount": 59718,
    "withheldInCountries": [],
    "affiliatesHighlightedLabel": {},
    "possiblySensitive": false,
    "pinnedTweetIds": [
      "1962212908840239366"
    ],
    "profile_bio": {
      "description": "OG GenAI Skeptic; spoke at US Senate. Warned about hallucinations in 2001. Advocating world models & neurosymbolic AI ever since. Author, Marcus on AI & 6 books",
      "entities": {
        "description": {},
        "url": {
          "urls": [
            {
              "display_url": "garymarcus.substack.com",
              "expanded_url": "http://garymarcus.substack.com",
              "indices": [
                0,
                23
              ],
              "url": "https://t.co/RrElrTVHSz"
            }
          ]
        }
      }
    },
    "isAutomated": false,
    "automatedBy": null
  },
  "extendedEntities": {},
  "card": null,
  "place": {},
  "entities": {
    "user_mentions": [
      {
        "id_str": "86626845",
        "indices": [
          3,
          13
        ],
        "name": "Eric Topol",
        "screen_name": "EricTopol"
      }
    ]
  },
  "quoted_tweet": null,
  "retweeted_tweet": {
    "type": "tweet",
    "id": "2070785799323533676",
    "url": "https://x.com/EricTopol/status/2070785799323533676",
    "twitterUrl": "https://twitter.com/EricTopol/status/2070785799323533676",
    "text": "Thanks for running our open-source work on current frontier models\n\n“The results are: the most capable models today (GPT-5.5 Pro) did outperform the best models from before (79/100 vs 69/100), but did not improve enough to be considered sufficient for reliable medical use.”\n\nRead full text and results below",
    "source": "Twitter for iPhone",
    "retweetCount": 42,
    "replyCount": 14,
    "likeCount": 285,
    "quoteCount": 4,
    "viewCount": 58769,
    "createdAt": "Sat Jun 27 08:26:22 +0000 2026",
    "lang": "en",
    "bookmarkCount": 90,
    "isReply": false,
    "inReplyToId": null,
    "conversationId": "2070785799323533676",
    "displayTextRange": [
      0,
      280
    ],
    "inReplyToUserId": null,
    "inReplyToUsername": null,
    "author": {
      "type": "user",
      "userName": "EricTopol",
      "url": "https://x.com/EricTopol",
      "twitterUrl": "https://twitter.com/EricTopol",
      "id": "86626845",
      "name": "Eric Topol",
      "isVerified": false,
      "isBlueVerified": true,
      "verifiedType": null,
      "profilePicture": "https://pbs.twimg.com/profile_images/1589325138960318464/2OwvQAWC_normal.jpg",
      "coverPicture": "https://pbs.twimg.com/profile_banners/86626845/1738777055",
      "description": "",
      "location": "La Jolla, CA",
      "followers": 778272,
      "following": 672,
      "status": "",
      "canDm": true,
      "canMediaTag": false,
      "createdAt": "Sun Nov 01 00:02:55 +0000 2009",
      "entities": {
        "description": {
          "urls": []
        },
        "url": {}
      },
      "fastFollowersCount": 0,
      "favouritesCount": 44364,
      "hasCustomTimelines": true,
      "isTranslator": false,
      "mediaCount": 32899,
      "statusesCount": 53405,
      "withheldInCountries": [],
      "affiliatesHighlightedLabel": {},
      "possiblySensitive": false,
      "pinnedTweetIds": [
        "1927470895343296519"
      ],
      "profile_bio": {
        "description": "physician-scientist, author of SUPER AGERS https://t.co/ZEdooyyJpP\nand Ground Truths: https://t.co/YhatcBT0hA",
        "entities": {
          "description": {
            "urls": [
              {
                "display_url": "tinyurl.com/3w76us9a",
                "expanded_url": "https://tinyurl.com/3w76us9a",
                "indices": [
                  43,
                  66
                ],
                "url": "https://t.co/ZEdooyyJpP"
              },
              {
                "display_url": "erictopol.substack.com",
                "expanded_url": "http://erictopol.substack.com",
                "indices": [
                  86,
                  109
                ],
                "url": "https://t.co/YhatcBT0hA"
              }
            ]
          },
          "url": {
            "urls": [
              {
                "display_url": "scripps.edu/translational",
                "expanded_url": "http://www.scripps.edu/translational",
                "indices": [
                  0,
                  23
                ],
                "url": "https://t.co/Z53npaOqS2"
              }
            ]
          }
        }
      },
      "isAutomated": false,
      "automatedBy": null
    },
    "extendedEntities": {},
    "card": null,
    "place": {},
    "entities": {
      "hashtags": [],
      "symbols": [],
      "urls": [],
      "user_mentions": []
    },
    "quoted_tweet": {
      "type": "tweet",
      "id": "2070742742133780960",
      "url": "https://x.com/yishan/status/2070742742133780960",
      "twitterUrl": "https://twitter.com/yishan/status/2070742742133780960",
      "text": "A big problem with research studies on AI models is that given how long the peer review process is, the results are always out-of-date by the time the paper is published.\n\nThis time, we have something better!\n\nThe typical reaction to research results like this roughly goes \"You're just testing on old models. Today's models are way better and surely can do it now!\"\n\nBut the best solution is for these papers to also open-source all of their testing framework so that upon publication, others can reproduce their results, as well as run it on the newest models of the day - and into the future. After all, \"this is the worst they'll ever be\" so what really matters is determining when they DO pass the threshold.\n\nAs it turns out, the authors of this paper DID open-source their evaluation framework!  \n\nHere:\nhttps://t.co/iXLwmItKwu\n\nSo I figured... let's re-run the tests on the latest models!\n\nSummary of our results are here:\nhttps://t.co/1Dzj0UcJUQ\n\nOne drawback is that, unfortunately, the authors didn't (or weren't legally able to) open-source ALL the testing data, since apparently some of it is copyrighted by JAMA/NEJM etc.  That's a separate problem with the medical research publishing industry for another time.\n\nHowever, we were able to reproduce the test on the public datasets they did include!\n\nFirst, we re-ran the same tests (as closely as we could) on the old models the paper claimed to use, in order to establish a baseline and determine how much \"drift\" there would be.  (Answer: not too much)\n\nThen we ran those tests on the newest frontier models we could find.  \n\nThe results are: the most capable models today (GPT-5.5 Pro) did outperform the best models from before (79/100 vs 69/100), but did not improve enough to be considered sufficient for reliable medical use.\n\nIn fact, the paper's criterion for \"fit for reliable medical use\" is more stringent, requiring the models to be robust under perturbation and bad data, knowing when to say there's not enough information, give clinically valid reasoning rather than hallucinations, etc.  Those sound pretty reasonable to me.\n\nI wasn't able to reproduce that kind of qualitative evaluation, but even on the basic pass/fail test using public datasets of interpreting radiology images, the newest models are better, but not yet quite good enough.\n\nNevertheless, I would like to praise the paper's authors for at least open-sourcing what they could, enabling me to (fairly quickly) attempt to reproduce their results.  This is definitely a step in the right direction!\n\nWhile my reproduction wasn't able to be comprehensive, it certainly gave me useful directional info and - perhaps more importantly - allowed me (a random dude on the internet) to directly reproduce the results in their paper and validate them.\n\nI would like to encourage ALL authors of research papers on AI models to do similar open-sourcing of their experimental frameworks!",
      "source": "Twitter for iPhone",
      "retweetCount": 75,
      "replyCount": 28,
      "likeCount": 603,
      "quoteCount": 15,
      "viewCount": 246657,
      "createdAt": "Sat Jun 27 05:35:16 +0000 2026",
      "lang": "en",
      "bookmarkCount": 137,
      "isReply": false,
      "inReplyToId": null,
      "conversationId": "2070742742133780960",
      "displayTextRange": [
        0,
        273
      ],
      "inReplyToUserId": null,
      "inReplyToUsername": null,
      "author": {
        "type": "user",
        "userName": "yishan",
        "url": "https://x.com/yishan",
        "twitterUrl": "https://twitter.com/yishan",
        "id": "14553823",
        "name": "Yishan",
        "isVerified": false,
        "isBlueVerified": true,
        "verifiedType": null,
        "profilePicture": "https://pbs.twimg.com/profile_images/887132881289662464/7sz43Ijt_normal.jpg",
        "coverPicture": "https://pbs.twimg.com/profile_banners/14553823/1500343937",
        "description": "",
        "location": "Made on Earth by Humans",
        "followers": 106088,
        "following": 534,
        "status": "",
        "canDm": true,
        "canMediaTag": true,
        "createdAt": "Sun Apr 27 01:12:38 +0000 2008",
        "entities": {
          "description": {
            "urls": []
          },
          "url": {}
        },
        "fastFollowersCount": 0,
        "favouritesCount": 10026,
        "hasCustomTimelines": true,
        "isTranslator": false,
        "mediaCount": 824,
        "statusesCount": 26416,
        "withheldInCountries": [],
        "affiliatesHighlightedLabel": {},
        "possiblySensitive": false,
        "pinnedTweetIds": [
          "2028193638103343317"
        ],
        "profile_bio": {
          "description": "I run Terraformation, and I was once the CEO of Reddit. Both are very interesting challenges.\n\nAMA in a subscriber-only newsletter! https://t.co/zA2F2S7etG",
          "entities": {
            "description": {
              "urls": [
                {
                  "display_url": "AskYishan.com",
                  "expanded_url": "https://AskYishan.com/",
                  "indices": [
                    132,
                    155
                  ],
                  "url": "https://t.co/zA2F2S7etG"
                }
              ]
            },
            "url": {
              "urls": [
                {
                  "display_url": "Terraformation.com",
                  "expanded_url": "http://www.Terraformation.com",
                  "indices": [
                    0,
                    23
                  ],
                  "url": "https://t.co/b8DYEnezHE"
                }
              ]
            }
          }
        },
        "isAutomated": false,
        "automatedBy": null
      },
      "extendedEntities": {},
      "card": null,
      "place": {},
      "entities": {
        "hashtags": [],
        "symbols": [],
        "urls": [
          {
            "display_url": "github.com/aiden-ygu/heal…",
            "expanded_url": "https://github.com/aiden-ygu/health-ai-readiness-eval/tree/v1.0.0",
            "indices": [
              811,
              834
            ],
            "url": "https://t.co/iXLwmItKwu"
          },
          {
            "display_url": "github.com/ywong137/healt…",
            "expanded_url": "https://github.com/ywong137/health-ai-readiness-vqarad-addendum",
            "indices": [
              931,
              954
            ],
            "url": "https://t.co/1Dzj0UcJUQ"
          }
        ],
        "user_mentions": []
      },
      "quoted_tweet": {
        "type": "tweet",
        "id": "2070436854072324140",
        "url": "",
        "twitterUrl": "",
        "text": "",
        "source": "Twitter for iPhone",
        "retweetCount": 0,
        "replyCount": 0,
        "likeCount": 0,
        "quoteCount": 0,
        "viewCount": 0,
        "createdAt": "",
        "lang": "",
        "bookmarkCount": 0,
        "isReply": false,
        "inReplyToId": null,
        "conversationId": "",
        "displayTextRange": [],
        "inReplyToUserId": null,
        "inReplyToUsername": null,
        "author": {},
        "extendedEntities": {},
        "card": null,
        "place": {},
        "entities": {},
        "quoted_tweet": null,
        "retweeted_tweet": null,
        "isLimitedReply": false,
        "communityInfo": null,
        "article": null
      },
      "retweeted_tweet": null,
      "isLimitedReply": false,
      "communityInfo": null,
      "article": null
    },
    "retweeted_tweet": null,
    "isLimitedReply": false,
    "communityInfo": null,
    "article": null
  },
  "isLimitedReply": false,
  "communityInfo": null,
  "article": null
}