Duplicate Bot: Discussions, second critic, more context (v5) (#61948)

Release Notes:

- N/A

script/github-track-duplicate-bot-effectiveness.py

@@ -8,8 +8,8 @@ Subcommands:
         Classify a closed issue and add it to the project board.
 
     classify-open
-        Classify open, triaged, bot-commented issues and add them to
-        the project board as Noise.
+        Classify open, triaged issues with a user-facing bot alert and add
+        them to the project board as Noise.
 
 Requires:
     requests (pip install requests)
@@ -36,14 +36,14 @@ REPO_NAME = "zed"
 STAFF_TEAM_SLUG = "staff"
 BOT_LOGIN = "zed-community-bot[bot]"
 BOT_APP_SLUG = "zed-community-bot"
-# Strings that identify a comment posted by the duplicate-detection bot. Any
-# match counts as a bot comment for classification purposes. A single comment
-# can contain both markers (v3+ produces this when there are both confident
-# duplicates and lower-confidence triage context).
-BOT_COMMENT_MARKERS = (
-    "This issue appears to be a duplicate of",  # user-facing duplicate alert
-    "Additional recent context for triagers",  # v3+ collapsed triage section
+ISSUE_DUPLICATE_ALERT_MARKER = "This issue appears to be a duplicate of"
+DISCUSSION_MATCH_ALERT_MARKER = "This looks like it may already be covered by an existing discussion"
+TRIAGE_CONTEXT_MARKER = "Additional recent context for triagers"
+USER_FACING_MATCH_MARKERS = (
+    ISSUE_DUPLICATE_ALERT_MARKER,
+    DISCUSSION_MATCH_ALERT_MARKER,
 )
+BOT_COMMENT_MARKERS = USER_FACING_MATCH_MARKERS + (TRIAGE_CONTEXT_MARKER,)
 BOT_START_DATE = "2026-02-18"
 NEEDS_TRIAGE_LABEL = "state:needs triage"
 DEFAULT_PROJECT_NUMBER = 76
@@ -54,6 +54,8 @@ TRANSIENT_HTTP_STATUSES = {429, 500, 502, 503, 504}
 # keep track of (e.g. the prompt gets a rewrite or the model gets swapped).
 # Newest first, please. The datetime is for the deployment time (merge to main).
 BOT_VERSION_TIMELINE = [
+    # Discussions, broader relevance-ranked retrieval, and critique for proposed matches.
+    ("v5", datetime(2026, 7, 30, 16, 38, tzinfo=timezone.utc)),
     ("v4", datetime(2026, 6, 16, 12, 43, tzinfo=timezone.utc)),
     ("v3", datetime(2026, 5, 25, 14, 30, tzinfo=timezone.utc)),
     ("v2", datetime(2026, 2, 26, 14, 9, tzinfo=timezone.utc)),
@@ -125,11 +127,15 @@ def is_bot_dupe_comment(body):
     return any(marker in body for marker in BOT_COMMENT_MARKERS)
 
 
+def has_user_facing_match_alert(body):
+    return any(marker in body for marker in USER_FACING_MATCH_MARKERS)
+
+
 def get_bot_comment_with_time(issue_number):
     """Get the bot's duplicate-detection comment and its timestamp from an issue.
 
-    Recognizes both the user-facing duplicate alert and the v3+ triage-only
-    comment formats. Returns {"body": str, "created_at": str} if found, else None.
+    Recognizes user-facing issue and Discussion alerts as well as triage-only
+    comments. Returns {"body": str, "created_at": str} if found, else None.
     """
     comments_path = f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/comments"
     page = 1
@@ -359,7 +365,12 @@ def classify_closed(issue_number, closer_login, state_reason):
 
     bot_comment = get_bot_comment_with_time(issue_number)
     bot_commented = bot_comment is not None
-    print(f"  Bot commented: {bot_commented}")
+    user_facing_alert = bot_commented and has_user_facing_match_alert(bot_comment["body"])
+    print(f"  Bot commented: {bot_commented}, user-facing alert: {user_facing_alert}")
+
+    if bot_commented and not user_facing_alert and state_reason != "duplicate":
+        print("  Skipping: triage-only context did not claim this was a duplicate")
+        return
 
     closer_is_author = closer_login == author
 
@@ -456,7 +467,7 @@ def classify_as_missed_opportunity(issue):
 
 
 def classify_open():
-    """Classify open, triaged, bot-commented issues as Noise."""
+    """Classify open, triaged issues with a user-facing bot alert as Noise."""
     print("Classifying open issues")
 
     query = (
@@ -478,13 +489,18 @@ def classify_open():
             author = (item.get("user") or {}).get("login", "")
             node_id = item["node_id"]
 
-            skip_reason = (
-                f"type is {type_name}" if type_name and type_name not in ("Bug", "Crash")
-                else f"author {author} is staff" if is_staff_member(author)
-                else "already on the board" if find_project_item(node_id)
-                else "no bot duplicate comment found" if not (bot_comment := get_bot_comment_with_time(number))
-                else None
-            )
+            if type_name and type_name not in ("Bug", "Crash"):
+                skip_reason = f"type is {type_name}"
+            elif is_staff_member(author):
+                skip_reason = f"author {author} is staff"
+            elif find_project_item(node_id):
+                skip_reason = "already on the board"
+            elif not (bot_comment := get_bot_comment_with_time(number)):
+                skip_reason = "no bot duplicate comment found"
+            elif not has_user_facing_match_alert(bot_comment["body"]):
+                skip_reason = "bot comment only contains triage context"
+            else:
+                skip_reason = None
 
             if skip_reason:
                 print(f"  #{number}: skipping, {skip_reason}")
@@ -520,7 +536,7 @@ if __name__ == "__main__":
 
     subparsers.add_parser(
         "classify-open",
-        help="Classify open, triaged, bot-commented issues as Noise.",
+        help="Classify open, triaged issues with a user-facing bot alert as Noise.",
     )
 
     args = parser.parse_args()