diff --git a/.github/workflows/comment_on_potential_duplicate_issues.yml b/.github/workflows/comment_on_potential_duplicate_issues.yml index 4b2cdd1fca..9509c3656e 100644 --- a/.github/workflows/comment_on_potential_duplicate_issues.yml +++ b/.github/workflows/comment_on_potential_duplicate_issues.yml @@ -31,7 +31,9 @@ jobs: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: - sparse-checkout: script/github-check-new-issue-for-duplicates.py + sparse-checkout: | + script/github-check-new-issue-for-duplicates.py + script/github_helpers.py sparse-checkout-cone-mode: false - name: Get github app token diff --git a/.github/workflows/community_pr_board.yml b/.github/workflows/community_pr_board.yml index 4cc9a5fac5..28d51ade6b 100644 --- a/.github/workflows/community_pr_board.yml +++ b/.github/workflows/community_pr_board.yml @@ -64,6 +64,7 @@ jobs: with: sparse-checkout: | script/github-community-pr-board.py + script/github_helpers.py script/community-pr-track-mapping.json sparse-checkout-cone-mode: false diff --git a/.github/workflows/community_pr_board_refresh.yml b/.github/workflows/community_pr_board_refresh.yml index f4b0c293cf..81540f3ace 100644 --- a/.github/workflows/community_pr_board_refresh.yml +++ b/.github/workflows/community_pr_board_refresh.yml @@ -42,6 +42,7 @@ jobs: with: sparse-checkout: | script/github-community-pr-board.py + script/github_helpers.py sparse-checkout-cone-mode: false - name: Set up Python diff --git a/.github/workflows/guild_assignment_status.yml b/.github/workflows/guild_assignment_status.yml index c91c05bb54..efd24a0f6b 100644 --- a/.github/workflows/guild_assignment_status.yml +++ b/.github/workflows/guild_assignment_status.yml @@ -45,6 +45,7 @@ jobs: with: sparse-checkout: | script/github-guild-board.py + script/github_helpers.py sparse-checkout-cone-mode: false - name: Set up Python diff --git a/.github/workflows/guild_new_pr_notify.yml b/.github/workflows/guild_new_pr_notify.yml index 52b4107bb2..f63839c504 100644 --- a/.github/workflows/guild_new_pr_notify.yml +++ b/.github/workflows/guild_new_pr_notify.yml @@ -45,6 +45,7 @@ jobs: with: sparse-checkout: | script/github-guild-board.py + script/github_helpers.py sparse-checkout-cone-mode: false - name: Set up Python diff --git a/.github/workflows/guild_stale_assignments.yml b/.github/workflows/guild_stale_assignments.yml index 81a7f0c0ff..d2cd5d6358 100644 --- a/.github/workflows/guild_stale_assignments.yml +++ b/.github/workflows/guild_stale_assignments.yml @@ -43,6 +43,7 @@ jobs: with: sparse-checkout: | script/github-guild-board.py + script/github_helpers.py sparse-checkout-cone-mode: false - name: Set up Python diff --git a/.github/workflows/guild_weekly_shipped.yml b/.github/workflows/guild_weekly_shipped.yml index 1f3bf71fea..dad19e1483 100644 --- a/.github/workflows/guild_weekly_shipped.yml +++ b/.github/workflows/guild_weekly_shipped.yml @@ -41,6 +41,7 @@ jobs: with: sparse-checkout: | script/github-guild-board.py + script/github_helpers.py sparse-checkout-cone-mode: false - name: Set up Python diff --git a/.github/workflows/track_duplicate_bot_effectiveness.yml b/.github/workflows/track_duplicate_bot_effectiveness.yml index 4ddf23e8bb..b7c5ae146a 100644 --- a/.github/workflows/track_duplicate_bot_effectiveness.yml +++ b/.github/workflows/track_duplicate_bot_effectiveness.yml @@ -25,7 +25,9 @@ jobs: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: - sparse-checkout: script/github-track-duplicate-bot-effectiveness.py + sparse-checkout: | + script/github-track-duplicate-bot-effectiveness.py + script/github_helpers.py sparse-checkout-cone-mode: false - name: Get github app token @@ -68,7 +70,9 @@ jobs: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: - sparse-checkout: script/github-track-duplicate-bot-effectiveness.py + sparse-checkout: | + script/github-track-duplicate-bot-effectiveness.py + script/github_helpers.py sparse-checkout-cone-mode: false - name: Get github app token diff --git a/.github/workflows/triage_queue_board.yml b/.github/workflows/triage_queue_board.yml index db80b4a22b..2898283baa 100644 --- a/.github/workflows/triage_queue_board.yml +++ b/.github/workflows/triage_queue_board.yml @@ -42,7 +42,9 @@ jobs: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: - sparse-checkout: script/github-triage-queue-board.py + sparse-checkout: | + script/github-triage-queue-board.py + script/github_helpers.py sparse-checkout-cone-mode: false persist-credentials: false diff --git a/script/github-check-new-issue-for-duplicates.py b/script/github-check-new-issue-for-duplicates.py index c1289c2156..902ee03085 100644 --- a/script/github-check-new-issue-for-duplicates.py +++ b/script/github-check-new-issue-for-duplicates.py @@ -32,12 +32,11 @@ import json import os import re import sys -import time from datetime import datetime, timedelta import requests +from github_helpers import github_graphql, github_rest_api, post_github_comment -GITHUB_API = "https://api.github.com" REPO_OWNER = "zed-industries" REPO_NAME = "zed" TRACKING_ISSUE_NUMBER = 46355 @@ -54,70 +53,26 @@ STOPWORDS = { "the", "this", "when", "while", "with", "won't", "work", "working", "zed", } -# HTTP statuses we'll retry on for GET requests -TRANSIENT_HTTP_STATUSES = {429, 500, 502, 503, 504} - def log(message): """Print to stderr so it doesn't interfere with JSON output on stdout.""" print(message, file=sys.stderr) -def github_api_get(path, params=None): - """Fetch JSON from the GitHub API, retrying transient failures. Raises on non-2xx status.""" - url = f"{GITHUB_API}/{path.lstrip('/')}" - for attempt in range(3): - try: - response = requests.get(url, headers=GITHUB_HEADERS, params=params) - response.raise_for_status() - return response.json() - except requests.RequestException as e: - transient = isinstance(e, (requests.ConnectionError, requests.Timeout)) or ( - isinstance(e, requests.HTTPError) and e.response.status_code in TRANSIENT_HTTP_STATUSES - ) - if not transient or attempt == 2: - raise - wait = 2 ** attempt - log(f" Transient GitHub API error ({e}); retrying in {wait}s") - time.sleep(wait) - - def github_search_issues(query, per_page=50, sort=None): """Search issues, using GitHub's relevance ordering unless a sort is specified.""" params = {"q": query, "per_page": per_page} if sort: params.update({"sort": sort, "order": "desc"}) - return github_api_get("/search/issues", params).get("items", []) - - -def github_api_graphql(query, variables=None): - """Run a GraphQL query against the GitHub API, retrying transient failures. """ - url = f"{GITHUB_API}/graphql" - for attempt in range(3): - try: - response = requests.post( - url, headers=GITHUB_HEADERS, json={"query": query, "variables": variables or {}} - ) - response.raise_for_status() - data = response.json() - if "errors" in data: - raise ValueError(f"GraphQL errors: {json.dumps(data['errors'])[:300]}") - return data["data"] - except requests.RequestException as e: - transient = isinstance(e, (requests.ConnectionError, requests.Timeout)) or ( - isinstance(e, requests.HTTPError) and e.response.status_code in TRANSIENT_HTTP_STATUSES - ) - if not transient or attempt == 2: - raise - wait = 2 ** attempt - log(f" Transient GitHub GraphQL error ({e}); retrying in {wait}s") - time.sleep(wait) + return github_rest_api("GET", "search/issues", params=params).get("items", []) def check_team_membership(org, team_slug, username): """Check if user is an active member of a team.""" try: - data = github_api_get(f"/orgs/{org}/teams/{team_slug}/memberships/{username}") + data = github_rest_api( + "GET", f"orgs/{org}/teams/{team_slug}/memberships/{username}" + ) return data.get("state") == "active" except requests.HTTPError as e: if e.response.status_code == 404: @@ -125,13 +80,6 @@ def check_team_membership(org, team_slug, username): raise -def post_comment(issue_number: int, body): - url = f"{GITHUB_API.rstrip('/')}/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/comments" - response = requests.post(url, headers=GITHUB_HEADERS, json={"body": body}) - response.raise_for_status() - log(f" Posted comment on #{issue_number}") - - def format_candidate_reference(match): candidate = match["candidate"] if candidate["kind"] == "discussion": @@ -288,7 +236,9 @@ def fetch_issue(issue_number: int): """Fetch issue from GitHub and return as a dict.""" log(f"Fetching issue #{issue_number}") - issue_data = github_api_get(f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}") + issue_data = github_rest_api( + "GET", f"repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}" + ) issue = { "number": issue_number, "title": issue_data["title"], @@ -320,8 +270,9 @@ def fetch_area_labels(): labels = [] page = 1 - while page_labels := github_api_get( - f"/repos/{REPO_OWNER}/{REPO_NAME}/labels", + while page_labels := github_rest_api( + "GET", + f"repos/{REPO_OWNER}/{REPO_NAME}/labels", params={"per_page": 100, "page": page}, ): labels.extend(page_labels) @@ -481,7 +432,9 @@ def parse_duplicate_magnets(): """ log(f"Parsing duplicate magnets from #{TRACKING_ISSUE_NUMBER}") - issue_data = github_api_get(f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{TRACKING_ISSUE_NUMBER}") + issue_data = github_rest_api( + "GET", f"repos/{REPO_OWNER}/{REPO_NAME}/issues/{TRACKING_ISSUE_NUMBER}" + ) body = issue_data.get("body") or "" # parse the issue body @@ -528,7 +481,9 @@ def enrich_magnets(magnets): """Fetch details for magnets and normalize them as candidates.""" log(f" Fetching details for {len(magnets)} magnets") for magnet in magnets: - data = github_api_get(f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{magnet['number']}") + data = github_rest_api( + "GET", f"repos/{REPO_OWNER}/{REPO_NAME}/issues/{magnet['number']}" + ) magnet.update({ "key": f"issue:{magnet['number']}", "kind": "issue", @@ -743,8 +698,9 @@ def enrich_popular_candidate_comments(candidates): ): continue try: - comments = github_api_get( - f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{candidate['number']}/comments", + comments = github_rest_api( + "GET", + f"repos/{REPO_OWNER}/{REPO_NAME}/issues/{candidate['number']}/comments", params={"per_page": 100}, ) except requests.RequestException as error: @@ -795,7 +751,7 @@ def search_discussions(issue, detected_areas, search_queries, max_searches=6): for search_type, query in queries[:max_searches]: log(f" Discussion search ({search_type}): {query}") try: - data = github_api_graphql(gql, {"q": query}) + data = github_graphql(gql, {"q": query}) for result_rank, node in enumerate(data["search"]["nodes"]): if not node: continue @@ -1322,22 +1278,12 @@ if __name__ == "__main__": parser.add_argument("--dry-run", action="store_true", help="Skip posting comment, just log what would be posted") args = parser.parse_args() - github_token = os.environ.get("GITHUB_TOKEN") anthropic_key = os.environ.get("ANTHROPIC_API_KEY") - if not github_token: - log("Error: GITHUB_TOKEN not set") - sys.exit(1) if not anthropic_key: log("Error: ANTHROPIC_API_KEY not set") sys.exit(1) - GITHUB_HEADERS = { - "Authorization": f"Bearer {github_token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } - issue = fetch_issue(args.issue_number) if should_skip(issue): print(json.dumps({"skipped": True})) @@ -1376,7 +1322,10 @@ if __name__ == "__main__": else: log("Posting comment") try: - post_comment(issue["number"], comment_body) + post_github_comment( + REPO_OWNER, REPO_NAME, issue["number"], comment_body + ) + log(f" Posted comment on #{issue['number']}") commented = True except requests.RequestException as e: log(f" Failed to post comment: {e}") diff --git a/script/github-community-pr-board.py b/script/github-community-pr-board.py index 495ee35c48..8c7231f005 100644 --- a/script/github-community-pr-board.py +++ b/script/github-community-pr-board.py @@ -46,16 +46,17 @@ Usage (called by the workflow, not directly): import json import os import sys -import time from pathlib import Path import requests +from github_helpers import ( + add_github_project_item, + fetch_github_project, + github_graphql, + github_rest_api, + set_github_project_field, +) -RETRYABLE_STATUS_CODES = {502, 503, 504} -MAX_RETRIES = 3 -RETRY_DELAY_SECONDS = 5 - -GITHUB_API_URL = "https://api.github.com" REPO_OWNER = "zed-industries" REPO_NAME = "zed" STAFF_TEAM_SLUG = "staff" @@ -81,7 +82,7 @@ def sync_track_from_labels(pr, project_number): pr_labels = {label["name"] for label in pr.get("labels", [])} track_name = resolve_track(pr_labels, load_mapping()) - project = github_fetch_project(project_number) + project = fetch_github_project(project_number) project_item = github_find_project_item(project["id"], pr["node_id"]) if not track_name: @@ -97,8 +98,8 @@ def sync_track_from_labels(pr, project_number): print(f"Resolved track: {track_name}") if not project_item: - project_item = github_add_to_project(project["id"], pr["node_id"]) - github_set_project_field(project, project_item, "Track", track_name) + project_item = add_github_project_item(project["id"], pr["node_id"]) + set_github_project_field(project, project_item, "Track", track_name) return project, project_item @@ -108,13 +109,13 @@ def set_progress_status_on_assignment(pr, assignee_login, project_number): print(f"Assignee '{assignee_login}' is not a staff member, skipping") return - project = github_fetch_project(project_number) + project = fetch_github_project(project_number) item_id = github_find_project_item(project["id"], pr["node_id"]) if not item_id: print(f"PR #{pr['number']} not on board, skipping assignment status update") return - github_set_project_field(project, item_id, "Status", STATUS_IN_PROGRESS_US) + set_github_project_field(project, item_id, "Status", STATUS_IN_PROGRESS_US) def return_to_reviewer(pr, project_number, reason): @@ -123,7 +124,7 @@ def return_to_reviewer(pr, project_number, reason): Called when the author signals they're ready for re-review, either by re-requesting review or by commenting on the PR. """ - project = github_fetch_project(project_number) + project = fetch_github_project(project_number) item_id = github_find_project_item(project["id"], pr["node_id"]) if not item_id: print(f"PR #{pr['number']} not on board, skipping") @@ -134,7 +135,7 @@ def return_to_reviewer(pr, project_number, reason): print( f"{reason}, flipping status from '{current_status}' to '{STATUS_IN_PROGRESS_US}'" ) - github_set_project_field(project, item_id, "Status", STATUS_IN_PROGRESS_US) + set_github_project_field(project, item_id, "Status", STATUS_IN_PROGRESS_US) else: print(f"Current status is '{current_status}', not flipping ({reason})") @@ -193,7 +194,7 @@ def recompute_signals(pr, project, project_item): def refresh_signals_if_on_board(pr, project_number): """Recompute signals for a PR only if it's already on the board.""" - project = github_fetch_project(project_number) + project = fetch_github_project(project_number) project_item = github_find_project_item(project["id"], pr["node_id"]) if not project_item: print(f"PR #{pr['number']} not on board, skipping signal refresh") @@ -207,7 +208,7 @@ def refresh_all_board_items(project_number): Backstop for the daily refresh workflow. Individual PR failures are logged and don't abort the rest of the run. """ - project = github_fetch_project(project_number) + project = fetch_github_project(project_number) processed = skipped = errors = 0 for item in github_list_project_items(project["id"]): if item.get("isArchived"): @@ -262,115 +263,28 @@ def resolve_track(pr_labels, tracks): return None -def github_graphql(query, variables): - """Execute a GitHub GraphQL query. Retries on transient server errors.""" - for attempt in range(MAX_RETRIES + 1): - response = requests.post( - f"{GITHUB_API_URL}/graphql", - headers=GITHUB_HEADERS, - json={"query": query, "variables": variables}, - ) - if response.status_code in RETRYABLE_STATUS_CODES and attempt < MAX_RETRIES: - print( - f"GitHub API returned {response.status_code}, retrying in {RETRY_DELAY_SECONDS}s (attempt {attempt + 1}/{MAX_RETRIES})..." - ) - time.sleep(RETRY_DELAY_SECONDS) - continue - response.raise_for_status() - result = response.json() - if "errors" in result: - raise RuntimeError(f"GraphQL error: {result['errors']}") - return result["data"] - raise RuntimeError("github_graphql: retry loop exited without return") - - -def github_rest_get(path): - """GET from the GitHub REST API. Retries on transient server errors.""" - for attempt in range(MAX_RETRIES + 1): - response = requests.get(f"{GITHUB_API_URL}/{path}", headers=GITHUB_HEADERS) - if response.status_code in RETRYABLE_STATUS_CODES and attempt < MAX_RETRIES: - print( - f"GitHub API returned {response.status_code}, retrying in {RETRY_DELAY_SECONDS}s (attempt {attempt + 1}/{MAX_RETRIES})..." - ) - time.sleep(RETRY_DELAY_SECONDS) - continue - response.raise_for_status() - return response.json() - raise RuntimeError("github_rest_get: retry loop exited without return") - - def github_is_staff_member(username): """Check if a user is a member of the staff team.""" try: - response = requests.get( - f"{GITHUB_API_URL}/orgs/{REPO_OWNER}/teams/{STAFF_TEAM_SLUG}/members/{username}", - headers=GITHUB_HEADERS, + github_rest_api( + "GET", f"orgs/{REPO_OWNER}/teams/{STAFF_TEAM_SLUG}/members/{username}" ) - if response.status_code == 204: - return True - if response.status_code == 404: + return True + except requests.HTTPError as error: + if error.response.status_code == 404: return False - print( - f"Warning: unexpected status {response.status_code} checking staff membership for '{username}'" - ) + print(f"Warning: failed to check staff membership for '{username}': {error}") return False - except requests.RequestException as exc: - print(f"Warning: failed to check staff membership for '{username}': {exc}") + except requests.RequestException as error: + print(f"Warning: failed to check staff membership for '{username}': {error}") return False def github_fetch_pr(pr_number): """Fetch a PR by number via the REST API.""" - return github_rest_get(f"repos/{REPO_OWNER}/{REPO_NAME}/pulls/{pr_number}") + return github_rest_api("GET", f"repos/{REPO_OWNER}/{REPO_NAME}/pulls/{pr_number}") -def github_fetch_project(project_number): - """Fetch a GitHub project board's metadata including fields and their options.""" - data = github_graphql( - """ - query($owner: String!, $number: Int!) { - organization(login: $owner) { - projectV2(number: $number) { - id - fields(first: 50) { - nodes { - ... on ProjectV2Field { - id - name - dataType - } - ... on ProjectV2SingleSelectField { - id - name - options { id name } - } - } - } - } - } - } - """, - {"owner": REPO_OWNER, "number": project_number}, - ) - return data["organization"]["projectV2"] - - -def github_add_to_project(project_id, content_node_id): - """Add a PR to the project board. Returns the new project item ID.""" - data = github_graphql( - """ - mutation($projectId: ID!, $contentId: ID!) { - addProjectV2ItemById(input: {projectId: $projectId, contentId: $contentId}) { - item { id } - } - } - """, - {"projectId": project_id, "contentId": content_node_id}, - ) - item_id = data["addProjectV2ItemById"]["item"]["id"] - print(f"Added PR to board (item: {item_id})") - return item_id - def github_list_project_items(project_id): """Yield every item on a project board, paginating as needed. @@ -563,58 +477,6 @@ def github_find_project_item(project_id, content_node_id): return None -def github_set_project_field(project, item_id, field_name, option_name): - """Set a single-select field on a project item.""" - field_id = None - option_id = None - for field in project["fields"]["nodes"]: - if field.get("name") == field_name: - field_id = field["id"] - for option in field.get("options", []): - if option["name"] == option_name: - option_id = option["id"] - break - break - - if not field_id: - available = [f["name"] for f in project["fields"]["nodes"] if "name" in f] - raise RuntimeError( - f"Field '{field_name}' not found on project. Available: {available}" - ) - if not option_id: - available = [ - opt["name"] - for f in project["fields"]["nodes"] - if f.get("name") == field_name - for opt in f.get("options", []) - ] - raise RuntimeError( - f"Option '{option_name}' not found in field '{field_name}'. " - f"Available: {available}" - ) - - github_graphql( - """ - mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { - updateProjectV2ItemFieldValue(input: { - projectId: $projectId - itemId: $itemId - fieldId: $fieldId - value: { singleSelectOptionId: $optionId } - }) { - projectV2Item { id } - } - } - """, - { - "projectId": project["id"], - "itemId": item_id, - "fieldId": field_id, - "optionId": option_id, - }, - ) - print(f"Set '{field_name}' to '{option_name}'") - def set_field_optional(project, item_id, field_name, option_name): """Set a single-select field, logging and skipping if the field or @@ -764,11 +626,6 @@ def github_get_field_value(item_id, field_name): if __name__ == "__main__": - GITHUB_HEADERS = { - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } project_number = int(os.environ["PROJECT_NUMBER"]) diff --git a/script/github-guild-board.py b/script/github-guild-board.py index 2635ac9b09..3202a2e39f 100644 --- a/script/github-guild-board.py +++ b/script/github-guild-board.py @@ -29,18 +29,20 @@ import html import json import os import random -import time import urllib.parse from datetime import datetime, timedelta, timezone from functools import lru_cache import requests +from github_helpers import ( + fetch_github_project, + github_graphql, + github_rest_api, + github_rest_get_paginated, + post_github_comment, + set_github_project_field, +) -RETRYABLE_STATUS_CODES = {502, 503, 504} -MAX_RETRIES = 3 -RETRY_DELAY_SECONDS = 5 - -GITHUB_API_URL = "https://api.github.com" REPO_OWNER = "zed-industries" REPO_NAME = "zed" # Cohort members are outside collaborators on the repo holding this custom @@ -95,71 +97,23 @@ ZEDGAR_QUIPS = [ ] -def github_graphql(query, variables): - for attempt in range(MAX_RETRIES + 1): - response = requests.post( - f"{GITHUB_API_URL}/graphql", - headers=GITHUB_HEADERS, - json={"query": query, "variables": variables}, - timeout=30, - ) - if response.status_code in RETRYABLE_STATUS_CODES and attempt < MAX_RETRIES: - time.sleep(RETRY_DELAY_SECONDS) - continue - response.raise_for_status() - result = response.json() - if "errors" in result: - raise RuntimeError(f"GraphQL error: {result['errors']}") - return result["data"] - raise RuntimeError("github_graphql: retry loop exited without return") - - -def github_rest_request(method, path, body=None): - url = f"{GITHUB_API_URL}/{path}" - for attempt in range(MAX_RETRIES + 1): - response = requests.request( - method, url, headers=GITHUB_HEADERS, json=body, timeout=30 - ) - if response.status_code in RETRYABLE_STATUS_CODES and attempt < MAX_RETRIES: - time.sleep(RETRY_DELAY_SECONDS) - continue - response.raise_for_status() - if response.status_code == 204 or not response.content: - return None - return response.json() - raise RuntimeError("github_rest_request: retry loop exited without return") - - -def github_rest_get_paginated(path): - results = [] - page = 1 - while True: - separator = "&" if "?" in path else "?" - batch = github_rest_request("GET", f"{path}{separator}per_page=100&page={page}") - if not batch: - break - results.extend(batch) - if len(batch) < 100: - break - page += 1 - return results - @lru_cache(maxsize=None) def is_guild_member(username): - response = requests.get( - f"{GITHUB_API_URL}/repos/{REPO_OWNER}/{REPO_NAME}/collaborators/{username}/permission", - headers=GITHUB_HEADERS, - timeout=30, - ) - # 404 means the user isn't a collaborator on the repo at all. - if response.status_code == 404: - return False - response.raise_for_status() + try: + response = github_rest_api( + "GET", + f"repos/{REPO_OWNER}/{REPO_NAME}/collaborators/{username}/permission", + ) + except requests.HTTPError as error: + # 404 means the user isn't a collaborator on the repo at all. + if error.response.status_code == 404: + return False + raise # role_name is the effective (highest) role for the user. For a cohort of # outside collaborators whose only grant is this custom role, that is the # custom role's name; built-in roles come back lowercased and won't match. - role_name = response.json().get("role_name") or "" + role_name = response.get("role_name") or "" return role_name.lower() == GUILD_ROLE_NAME.lower() @@ -231,30 +185,6 @@ def pr_closing_issues(pr_node_id): return (node.get("closingIssuesReferences") or {}).get("nodes", []) -def fetch_project(project_number): - data = github_graphql( - """ - query($owner: String!, $number: Int!) { - organization(login: $owner) { - projectV2(number: $number) { - id - fields(first: 50) { - nodes { - ... on ProjectV2Field { id name dataType } - ... on ProjectV2SingleSelectField { id name options { id name } } - } - } - } - } - } - """, - {"owner": REPO_OWNER, "number": project_number}, - ) - project = data["organization"]["projectV2"] - if not project: - raise RuntimeError(f"Project #{project_number} not found in {REPO_OWNER}") - return project - def find_project_item(project_id, content_node_id): # Includes each item's single-select values so callers can read Status @@ -291,34 +221,6 @@ def find_project_item(project_id, content_node_id): return None -def set_project_field(project, item_id, field_name, option_name): - field = next( - (f for f in project["fields"]["nodes"] if f.get("name") == field_name), None - ) - if not field: - raise RuntimeError(f"Field '{field_name}' not found on board") - option_id = next( - (o["id"] for o in field.get("options", []) if o["name"] == option_name), None - ) - if not option_id: - raise RuntimeError(f"Option '{option_name}' not found in field '{field_name}'") - github_graphql( - """ - mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { - updateProjectV2ItemFieldValue(input: { - projectId: $projectId, itemId: $itemId, fieldId: $fieldId, - value: { singleSelectOptionId: $optionId } - }) { projectV2Item { id } } - } - """, - { - "projectId": project["id"], - "itemId": item_id, - "fieldId": field["id"], - "optionId": option_id, - }, - ) - def list_project_issues(project_id): cursor = None @@ -365,17 +267,11 @@ def list_project_issues(project_id): cursor = page["pageInfo"]["endCursor"] -def post_comment(issue_number, body): - github_rest_request( - "POST", f"repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/comments", {"body": body} - ) - - def remove_assignees(issue_number, assignees): - github_rest_request( + github_rest_api( "DELETE", f"repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/assignees", - {"assignees": assignees}, + body={"assignees": assignees}, ) @@ -441,7 +337,7 @@ def handle_assignment(issue, assignee_login, project_number): if not is_guild_member(assignee_login): return - project = fetch_project(project_number) + project = fetch_github_project(project_number) item = find_project_item(project["id"], issue["node_id"]) if not item: send_slack( @@ -452,7 +348,7 @@ def handle_assignment(issue, assignee_login, project_number): ) return - set_project_field(project, item["id"], STATUS_FIELD, STATUS_IN_PROGRESS) + set_github_project_field(project, item["id"], STATUS_FIELD, STATUS_IN_PROGRESS) def handle_unassignment(issue, removed_login, sender, project_number): @@ -462,7 +358,7 @@ def handle_unassignment(issue, removed_login, sender, project_number): if not is_guild_member(removed_login): return - project = fetch_project(project_number) + project = fetch_github_project(project_number) item = find_project_item(project["id"], issue["node_id"]) if not item or item_status(item) != STATUS_IN_PROGRESS: return @@ -476,7 +372,7 @@ def handle_unassignment(issue, removed_login, sender, project_number): status = STATUS_FOR_TYPE.get((issue.get("type") or {}).get("name") or "") link = slack_link(issue["html_url"], issue["title"]) if status: - set_project_field(project, item["id"], STATUS_FIELD, status) + set_github_project_field(project, item["id"], STATUS_FIELD, status) send_slack( f"@{removed_login} {who} In Progress issue #{issue['number']} " f"{link}. I moved it back to *{status}*, " @@ -509,11 +405,11 @@ def set_linked_issues_in_progress(pull_request, project_number): ] if not open_issues: return - project = fetch_project(project_number) + project = fetch_github_project(project_number) for issue in open_issues: item = find_project_item(project["id"], issue["id"]) if item: - set_project_field(project, item["id"], STATUS_FIELD, STATUS_IN_PROGRESS) + set_github_project_field(project, item["id"], STATUS_FIELD, STATUS_IN_PROGRESS) def warn_about_extra_open_prs(pull_request, author): @@ -521,7 +417,7 @@ def warn_about_extra_open_prs(pull_request, author): f"repo:{REPO_OWNER}/{REPO_NAME} is:pr is:open " f"author:{author} created:>={COHORT_START_DATE}" ) - result = github_rest_request( + result = github_rest_api( "GET", f"search/issues?q={urllib.parse.quote(query)}&per_page=100" ) others = [ @@ -589,7 +485,7 @@ def run_event(project_number): def run_stale(project_number): - project = fetch_project(project_number) + project = fetch_github_project(project_number) checked_in = cleared = 0 for item in list_project_issues(project["id"]): @@ -638,7 +534,9 @@ def run_stale(project_number): # they have stayed quiet for the check-in window. if (NOW - last_activity).days < CHECK_IN_AFTER_DAYS: continue - post_comment( + post_github_comment( + REPO_OWNER, + REPO_NAME, issue_number, CHECK_IN_BODY.format( marker=CHECK_IN_MARKER, @@ -658,7 +556,7 @@ def run_stale(project_number): status = STATUS_FOR_TYPE.get((content.get("issueType") or {}).get("name") or "") link = slack_link(content["url"], content["title"]) if status: - set_project_field(project, item["id"], STATUS_FIELD, status) + set_github_project_field(project, item["id"], STATUS_FIELD, status) send_slack( f"Issue #{issue_number} {link} went quiet with @{assignee} " "(no open PR, no reply to my check-in), so I cleared the assignment and " @@ -678,7 +576,7 @@ def run_stale(project_number): def run_weekly(project_number): - project = fetch_project(project_number) + project = fetch_github_project(project_number) cutoff = NOW - timedelta(days=SHIPPED_WINDOW_DAYS) shipped = [] @@ -705,11 +603,6 @@ def run_weekly(project_number): if __name__ == "__main__": - GITHUB_HEADERS = { - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } NOW = datetime.now(timezone.utc) diff --git a/script/github-track-duplicate-bot-effectiveness.py b/script/github-track-duplicate-bot-effectiveness.py index a01a8941eb..569ccb2cff 100644 --- a/script/github-track-duplicate-bot-effectiveness.py +++ b/script/github-track-duplicate-bot-effectiveness.py @@ -24,13 +24,11 @@ import functools import os import re import sys -import time from datetime import datetime, timezone import requests +from github_helpers import add_github_project_item, github_graphql, github_rest_api -GITHUB_API = "https://api.github.com" -GRAPHQL_URL = "https://api.github.com/graphql" REPO_OWNER = "zed-industries" REPO_NAME = "zed" STAFF_TEAM_SLUG = "staff" @@ -48,8 +46,7 @@ BOT_START_DATE = "2026-02-18" NEEDS_TRIAGE_LABEL = "state:needs triage" DEFAULT_PROJECT_NUMBER = 76 VALID_CLOSED_AS_VALUES = {"duplicate", "not_planned", "completed"} -# HTTP statuses we'll retry on for GET requests -TRANSIENT_HTTP_STATUSES = {429, 500, 502, 503, 504} + # Add a new tuple when you deploy a new version of the bot that you want to # keep track of (e.g. the prompt gets a rewrite or the model gets swapped). # Newest first, please. The datetime is for the deployment time (merge to main). @@ -74,37 +71,19 @@ def bot_version_for_time(date_string): return BOT_VERSION_TIMELINE[-1][0] -def github_api_get(path, params=None): - """Fetch JSON from the GitHub REST API, retrying transient failures. Raises on non-2xx status.""" - url = f"{GITHUB_API}/{path.lstrip('/')}" - for attempt in range(3): - try: - response = requests.get(url, headers=GITHUB_HEADERS, params=params) - response.raise_for_status() - return response.json() - except requests.RequestException as e: - transient = isinstance(e, (requests.ConnectionError, requests.Timeout)) or ( - isinstance(e, requests.HTTPError) and e.response.status_code in TRANSIENT_HTTP_STATUSES - ) - if not transient or attempt == 2: - raise - wait = 2 ** attempt - print(f" Transient GitHub API error ({e}); retrying in {wait}s") - time.sleep(wait) - - def github_search_issues(query): """Search issues, returning most recently created first.""" # not handling pagination on purpose: the oldest issues are on the board already params = {"q": query, "sort": "created", "order": "desc", "per_page": 100} - return github_api_get("/search/issues", params).get("items", []) + return github_rest_api("GET", "search/issues", params=params).get("items", []) def is_staff_member(username): """Check if user is an active member of the staff team.""" try: - data = github_api_get( - f"/orgs/{REPO_OWNER}/teams/{STAFF_TEAM_SLUG}/memberships/{username}" + data = github_rest_api( + "GET", + f"orgs/{REPO_OWNER}/teams/{STAFF_TEAM_SLUG}/memberships/{username}", ) return data.get("state") == "active" except requests.HTTPError as error: @@ -114,7 +93,9 @@ def is_staff_member(username): def fetch_issue(issue_number): - data = github_api_get(f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}") + data = github_rest_api( + "GET", f"repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}" + ) return { "number": issue_number, "node_id": data["node_id"], @@ -139,9 +120,11 @@ def get_bot_comment_with_time(issue_number): Recognizes user-facing issue and Discussion alerts as well as triage-only comments. Returns {"body": str, "created_at": str} if found, else None. """ - comments_path = f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/comments" + comments_path = f"repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/comments" page = 1 - while comments := github_api_get(comments_path, {"per_page": 100, "page": page}): + while comments := github_rest_api( + "GET", comments_path, params={"per_page": 100, "page": page} + ): for comment in comments: author = (comment.get("user") or {}).get("login", "") body = comment.get("body", "") @@ -156,22 +139,6 @@ def parse_suggested_issues(comment_body): return [int(match) for match in re.findall(r"^- #(\d+)", comment_body, re.MULTILINE)] -def github_api_graphql(query, variables=None, partial_errors_ok=False): - """Execute a GitHub GraphQL query. Raises on errors unless partial_errors_ok is set.""" - response = requests.post( - GRAPHQL_URL, - headers=GITHUB_HEADERS, - json={"query": query, "variables": variables or {}}, - ) - response.raise_for_status() - data = response.json() - if "errors" in data: - if not partial_errors_ok or "data" not in data: - raise RuntimeError(f"GraphQL errors: {data['errors']}") - print(f" GraphQL partial errors (ignored): {data['errors']}") - return data["data"] - - def find_canonical_among(duplicate_number, candidates): """Check if any candidate issue has duplicate_number marked as a duplicate. @@ -187,7 +154,7 @@ def find_canonical_among(duplicate_number, candidates): # candidate issue numbers are baked into the query body via field aliases # (GraphQL doesn't let you parametrize alias names), so $numbers isn't needed. - data = github_api_graphql( + data = github_graphql( """ query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { @@ -219,7 +186,7 @@ def find_canonical_among(duplicate_number, candidates): @functools.lru_cache def get_project_config(): """Fetch the project board's ID, field IDs, and option IDs.""" - data = github_api_graphql( + data = github_graphql( """ query($org: String!, $number: Int!) { organization(login: $org) { @@ -260,7 +227,7 @@ def find_project_item(issue_node_id): Returns the project item ID if found, or None. """ - data = github_api_graphql( + data = github_graphql( "query($id: ID!) { node(id: $id) { ... on Issue { projectItems(first: 20) { nodes { id project { number } } } } } }", {"id": issue_node_id}, ) @@ -273,17 +240,7 @@ def find_project_item(issue_node_id): def add_project_item(issue_node_id): """Add an issue to the project board. Returns the new item ID.""" config = get_project_config() - data = github_api_graphql( - """ - mutation($projectId: ID!, $contentId: ID!) { - addProjectV2ItemById(input: {projectId: $projectId, contentId: $contentId}) { - item { id } - } - } - """, - {"projectId": config["project_id"], "contentId": issue_node_id}, - ) - return data["addProjectV2ItemById"]["item"]["id"] + return add_github_project_item(config["project_id"], issue_node_id) def set_field_value(item_id, field_name, value): @@ -305,7 +262,7 @@ def set_field_value(item_id, field_name, value): # text field field_value = {"text": str(value)} - github_api_graphql( + github_graphql( """ mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $value: ProjectV2FieldValue!) { updateProjectV2ItemFieldValue(input: { @@ -543,11 +500,6 @@ if __name__ == "__main__": args = parser.parse_args() - GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "") - if not GITHUB_TOKEN: - print("Error: GITHUB_TOKEN environment variable is required") - sys.exit(1) - raw_project_number = os.environ.get("PROJECT_NUMBER", "") if raw_project_number: try: @@ -558,11 +510,6 @@ if __name__ == "__main__": else: PROJECT_NUMBER = DEFAULT_PROJECT_NUMBER - GITHUB_HEADERS = { - "Authorization": f"token {GITHUB_TOKEN}", - "Accept": "application/vnd.github+json", - } - if args.command == "classify-closed": classify_closed(args.issue_number, args.closer_login, args.state_reason) elif args.command == "classify-open": diff --git a/script/github-triage-queue-board.py b/script/github-triage-queue-board.py index 73738cd302..caebba03bb 100644 --- a/script/github-triage-queue-board.py +++ b/script/github-triage-queue-board.py @@ -15,18 +15,12 @@ Usage (called by the workflow, not directly): """ import os -import time import urllib.parse from datetime import datetime, timedelta from zoneinfo import ZoneInfo -import requests +from github_helpers import add_github_project_item, github_graphql, github_rest_api -RETRYABLE_STATUS_CODES = {502, 503, 504} -MAX_RETRIES = 3 -RETRY_DELAY_SECONDS = 5 - -GITHUB_API_URL = "https://api.github.com" REPO_OWNER = "zed-industries" REPO_NAME = "zed" # The eligibility window's boundaries are calendar days in this timezone. GitHub @@ -36,44 +30,6 @@ REPO_NAME = "zed" BUSINESS_DAY_TZ = ZoneInfo("America/Los_Angeles") -def github_graphql(query, variables): - for attempt in range(MAX_RETRIES + 1): - response = requests.post( - f"{GITHUB_API_URL}/graphql", - headers=GITHUB_HEADERS, - json={"query": query, "variables": variables}, - timeout=30, - ) - if response.status_code in RETRYABLE_STATUS_CODES and attempt < MAX_RETRIES: - print( - f"GitHub API returned {response.status_code}, retrying in {RETRY_DELAY_SECONDS}s (attempt {attempt + 1}/{MAX_RETRIES})..." - ) - time.sleep(RETRY_DELAY_SECONDS) - continue - response.raise_for_status() - result = response.json() - if "errors" in result: - raise RuntimeError(f"GraphQL error: {result['errors']}") - return result["data"] - raise RuntimeError("github_graphql: retry loop exited without return") - - -def github_rest_get(path): - for attempt in range(MAX_RETRIES + 1): - response = requests.get( - f"{GITHUB_API_URL}/{path}", headers=GITHUB_HEADERS, timeout=30 - ) - if response.status_code in RETRYABLE_STATUS_CODES and attempt < MAX_RETRIES: - print( - f"GitHub API returned {response.status_code}, retrying in {RETRY_DELAY_SECONDS}s (attempt {attempt + 1}/{MAX_RETRIES})..." - ) - time.sleep(RETRY_DELAY_SECONDS) - continue - response.raise_for_status() - return response.json() - raise RuntimeError("github_rest_get: retry loop exited without return") - - def day_start(day): # First instant of a calendar day in the business-day timezone. return datetime(day.year, day.month, day.day, 0, 0, 0, tzinfo=BUSINESS_DAY_TZ) @@ -108,8 +64,8 @@ def eligible_issues(window_start, window_end): issues = {} page = 1 while True: - data = github_rest_get( - f"search/issues?q={urllib.parse.quote(query)}&per_page=100&page={page}" + data = github_rest_api( + "GET", f"search/issues?q={urllib.parse.quote(query)}&per_page=100&page={page}" ) for item in data["items"]: issues[item["node_id"]] = item["number"] @@ -165,13 +121,6 @@ def project_items(project_id): cursor = items["pageInfo"]["endCursor"] -def add_to_project(project_id, content_id): - github_graphql( - "mutation($p: ID!, $c: ID!) { addProjectV2ItemById(input: {projectId: $p, contentId: $c}) { item { id } } }", - {"p": project_id, "c": content_id}, - ) - - def remove_from_project(project_id, item_id): github_graphql( "mutation($p: ID!, $i: ID!) { deleteProjectV2Item(input: {projectId: $p, itemId: $i}) { deletedItemId } }", @@ -192,7 +141,7 @@ def sync_project(project_id, dry_run=False): if content_id not in on_board: print(f"+ #{number}") if not dry_run: - add_to_project(project_id, content_id) + add_github_project_item(project_id, content_id) for item_id, content_id, number in current: if content_id not in eligible: @@ -202,11 +151,6 @@ def sync_project(project_id, dry_run=False): if __name__ == "__main__": - GITHUB_HEADERS = { - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } project_number = int(os.environ["PROJECT_NUMBER"]) project_id = fetch_project_id(project_number) sync_project(project_id, os.environ.get("DRY_RUN") == "true") diff --git a/script/github_helpers.py b/script/github_helpers.py new file mode 100644 index 0000000000..42826d36c2 --- /dev/null +++ b/script/github_helpers.py @@ -0,0 +1,214 @@ +import os +import sys +import time + +import requests + +GITHUB_API_URL = "https://api.github.com" +RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504} +MAX_ATTEMPTS = 4 +REQUEST_TIMEOUT_SECONDS = 30 + + +def _github_headers(): + token = os.environ.get("GITHUB_TOKEN") + if not token: + raise RuntimeError("GITHUB_TOKEN environment variable is required") + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def _retry_delay(response, attempt): + retry_after = response.headers.get("Retry-After") + if retry_after: + try: + return max(float(retry_after), 0) + except ValueError: + pass + return 2**attempt + + +def _github_request(method, path, *, params=None, body=None, retry=True): + url = f"{GITHUB_API_URL}/{path}" + max_attempts = MAX_ATTEMPTS if retry else 1 + for attempt in range(max_attempts): + try: + response = requests.request( + method, + url, + headers=_github_headers(), + params=params, + json=body, + timeout=REQUEST_TIMEOUT_SECONDS, + ) + except (requests.ConnectionError, requests.Timeout) as error: + if attempt == max_attempts - 1: + raise + delay = 2**attempt + print( + f"Transient GitHub API error ({error}); retrying in {delay}s", + file=sys.stderr, + ) + time.sleep(delay) + continue + + if ( + response.status_code in RETRYABLE_STATUS_CODES + and attempt < max_attempts - 1 + ): + delay = _retry_delay(response, attempt) + print( + f"GitHub API returned {response.status_code}; retrying in {delay}s", + file=sys.stderr, + ) + time.sleep(delay) + continue + + response.raise_for_status() + return response + + raise RuntimeError("GitHub API retry loop exited without returning") + + +def github_rest_api(method, path, params=None, body=None): + # Retrying writes after an ambiguous timeout can duplicate comments or mutations. + retry = method.upper() in {"GET", "HEAD", "OPTIONS"} + response = _github_request( + method, path, params=params, body=body, retry=retry + ) + if response.status_code == 204 or not response.content: + return {} + return response.json() + + +def github_rest_get_paginated(path): + results = [] + page = 1 + while True: + batch = github_rest_api( + "GET", path, params={"per_page": 100, "page": page} + ) + if not batch: + return results + results.extend(batch) + if len(batch) < 100: + return results + page += 1 + + +def post_github_comment(owner, repository, issue_number, body): + github_rest_api( + "POST", + f"repos/{owner}/{repository}/issues/{issue_number}/comments", + body={"body": body}, + ) + + +def github_graphql(query, variables=None, *, partial_errors_ok=False): + # GraphQL uses POST for reads and writes, so classify the operation before retrying. + retry = not query.lstrip().startswith("mutation") + response = _github_request( + "POST", + "graphql", + body={"query": query, "variables": variables or {}}, + retry=retry, + ) + result = response.json() + if "errors" in result: + if not partial_errors_ok or "data" not in result: + raise RuntimeError(f"GraphQL errors: {result['errors']}") + print(f"GraphQL partial errors (ignored): {result['errors']}", file=sys.stderr) + return result["data"] + + +def fetch_github_project(project_number, owner="zed-industries"): + data = github_graphql( + """ + query($owner: String!, $number: Int!) { + organization(login: $owner) { + projectV2(number: $number) { + id + fields(first: 50) { + nodes { + ... on ProjectV2Field { id name dataType } + ... on ProjectV2SingleSelectField { id name options { id name } } + } + } + } + } + } + """, + {"owner": owner, "number": project_number}, + ) + project = data["organization"]["projectV2"] + if not project: + raise RuntimeError(f"Project #{project_number} not found in {owner}") + return project + + +def add_github_project_item(project_id, content_id): + data = github_graphql( + """ + mutation($projectId: ID!, $contentId: ID!) { + addProjectV2ItemById(input: {projectId: $projectId, contentId: $contentId}) { + item { id } + } + } + """, + {"projectId": project_id, "contentId": content_id}, + ) + return data["addProjectV2ItemById"]["item"]["id"] + + +def set_github_project_field(project, item_id, field_name, option_name): + field = next( + (field for field in project["fields"]["nodes"] if field.get("name") == field_name), + None, + ) + if not field: + available = [ + field["name"] for field in project["fields"]["nodes"] if "name" in field + ] + raise RuntimeError( + f"Field '{field_name}' not found on project. Available: {available}" + ) + + option_id = next( + ( + option["id"] + for option in field.get("options", []) + if option["name"] == option_name + ), + None, + ) + if not option_id: + available = [option["name"] for option in field.get("options", [])] + raise RuntimeError( + f"Option '{option_name}' not found in field '{field_name}'. " + f"Available: {available}" + ) + + github_graphql( + """ + mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { + updateProjectV2ItemFieldValue(input: { + projectId: $projectId + itemId: $itemId + fieldId: $fieldId + value: { singleSelectOptionId: $optionId } + }) { + projectV2Item { id } + } + } + """, + { + "projectId": project["id"], + "itemId": item_id, + "fieldId": field["id"], + "optionId": option_id, + }, + ) + print(f"Set '{field_name}' to '{option_name}'")