github.js
18,649 bytes
| 1 | const API = "https://api.github.com"; |
|---|---|
| 2 | |
| 3 | async function request(path, token, options = {}) { |
| 4 | let response; |
| 5 | for (let attempt = 0; attempt < 3; attempt += 1) { |
| 6 | response = await fetch(`${API}${path}`, { |
| 7 | ...options, |
| 8 | signal: options.signal || AbortSignal.timeout(20_000), |
| 9 | headers: { |
| 10 | Accept: "application/vnd.github+json", |
| 11 | Authorization: `Bearer ${token}`, |
| 12 | "X-GitHub-Api-Version": "2022-11-28", |
| 13 | "Content-Type": "application/json", |
| 14 | ...options.headers, |
| 15 | }, |
| 16 | }); |
| 17 | if (![429, 502, 503, 504].includes(response.status) || attempt === 2) break; |
| 18 | await new Promise((resolve) => setTimeout(resolve, 250 * (attempt + 1))); |
| 19 | } |
| 20 | if (!response.ok) { |
| 21 | const detail = await response.text(); |
| 22 | if (response.status === 403 && response.headers.get("x-ratelimit-remaining") === "0") { |
| 23 | const error = new Error("GitHub API rate limit reached. Try creating the snapshot after the reset time."); |
| 24 | error.publicMessage = error.message; |
| 25 | throw error; |
| 26 | } |
| 27 | if (response.status === 403) { |
| 28 | const error = new Error(`GitHub request failed (${response.status}): ${detail}`); |
| 29 | error.publicMessage = "GitHub denied access. Grant the GitHub App at least read-only Contents and Metadata, approve the updated installation, then try again."; |
| 30 | error.forbidden = true; |
| 31 | throw error; |
| 32 | } |
| 33 | const error = new Error(`GitHub request failed (${response.status}): ${detail}`); |
| 34 | if (response.status === 404) error.missing = true; |
| 35 | throw error; |
| 36 | } |
| 37 | return response.json(); |
| 38 | } |
| 39 | |
| 40 | /** |
| 41 | * For the parts of a snapshot that are nice to have rather than required. |
| 42 | * |
| 43 | * Issues and pull requests need their own read permissions, and a repository |
| 44 | * can have issues switched off entirely. Neither is a reason to fail the whole |
| 45 | * snapshot: the code is what the snapshot is for, and every view already |
| 46 | * renders an empty list as "none". A rate-limit 403 still throws, because that |
| 47 | * one is temporary and silently producing a half-empty snapshot would hide it. |
| 48 | */ |
| 49 | async function optional(promise, fallback) { |
| 50 | try { |
| 51 | return await promise; |
| 52 | } catch (error) { |
| 53 | if (error.forbidden || error.missing) return fallback; |
| 54 | throw error; |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | async function allPages(path, token) { |
| 59 | const rows = []; |
| 60 | for (let page = 1; ; page += 1) { |
| 61 | const separator = path.includes("?") ? "&" : "?"; |
| 62 | const result = await request(`${path}${separator}per_page=100&page=${page}`, token); |
| 63 | const pageRows = Array.isArray(result) ? result : result.repositories; |
| 64 | rows.push(...pageRows); |
| 65 | if (pageRows.length < 100) return rows; |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | async function mapLimit(items, limit, worker) { |
| 70 | const results = new Array(items.length); |
| 71 | let cursor = 0; |
| 72 | async function run() { |
| 73 | while (cursor < items.length) { |
| 74 | const index = cursor; |
| 75 | cursor += 1; |
| 76 | results[index] = await worker(items[index], index); |
| 77 | } |
| 78 | } |
| 79 | await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run)); |
| 80 | return results; |
| 81 | } |
| 82 | |
| 83 | async function completeTree(root, branch, token) { |
| 84 | const recursive = await request(`${root}/git/trees/${encodeURIComponent(branch)}?recursive=1`, token); |
| 85 | if (!recursive.truncated) return recursive.tree; |
| 86 | |
| 87 | const files = []; |
| 88 | const pending = [{ sha: branch, prefix: "" }]; |
| 89 | while (pending.length) { |
| 90 | const batch = pending.splice(0, 8); |
| 91 | const trees = await Promise.all(batch.map((item) => request( |
| 92 | `${root}/git/trees/${encodeURIComponent(item.sha)}`, |
| 93 | token, |
| 94 | ).then((tree) => ({ ...item, tree: tree.tree })))); |
| 95 | for (const current of trees) { |
| 96 | for (const entry of current.tree) { |
| 97 | const path = current.prefix ? `${current.prefix}/${entry.path}` : entry.path; |
| 98 | if (entry.type === "tree") pending.push({ sha: entry.sha, prefix: path }); |
| 99 | else files.push({ ...entry, path }); |
| 100 | } |
| 101 | } |
| 102 | } |
| 103 | return files; |
| 104 | } |
| 105 | |
| 106 | async function commitWithAllFiles(root, sha, token) { |
| 107 | let detail; |
| 108 | const files = []; |
| 109 | for (let page = 1; page <= 30; page += 1) { |
| 110 | const current = await request( |
| 111 | `${root}/commits/${encodeURIComponent(sha)}?per_page=100&page=${page}`, |
| 112 | token, |
| 113 | ); |
| 114 | if (!detail) detail = current; |
| 115 | files.push(...(current.files || [])); |
| 116 | if (!current.files || current.files.length < 100) break; |
| 117 | } |
| 118 | return { ...detail, files }; |
| 119 | } |
| 120 | |
| 121 | function decodeFile(content) { |
| 122 | const buffer = Buffer.from(content.replace(/\n/g, ""), "base64"); |
| 123 | if (buffer.includes(0)) return null; |
| 124 | return buffer.toString("utf8"); |
| 125 | } |
| 126 | |
| 127 | function imageMediaType(path) { |
| 128 | const extension = path.toLocaleLowerCase("en").split(".").pop(); |
| 129 | return { |
| 130 | png: "image/png", |
| 131 | jpg: "image/jpeg", |
| 132 | jpeg: "image/jpeg", |
| 133 | gif: "image/gif", |
| 134 | webp: "image/webp", |
| 135 | }[extension] || null; |
| 136 | } |
| 137 | |
| 138 | function account(user) { |
| 139 | return { |
| 140 | login: user?.login || "ghost", |
| 141 | avatarUrl: user?.avatar_url || "", |
| 142 | }; |
| 143 | } |
| 144 | |
| 145 | function conversationComment(comment, type = "comment") { |
| 146 | return { |
| 147 | id: comment.id, |
| 148 | type, |
| 149 | body: comment.body || "", |
| 150 | author: account(comment.user), |
| 151 | createdAt: comment.created_at || comment.submitted_at, |
| 152 | updatedAt: comment.updated_at || comment.submitted_at, |
| 153 | state: comment.state || null, |
| 154 | path: comment.path || null, |
| 155 | line: comment.line || comment.original_line || null, |
| 156 | }; |
| 157 | } |
| 158 | |
| 159 | export function createGitHubClient(config) { |
| 160 | async function requestUserToken(body) { |
| 161 | let response; |
| 162 | try { |
| 163 | response = await fetch("https://github.com/login/oauth/access_token", { |
| 164 | method: "POST", |
| 165 | signal: AbortSignal.timeout(20_000), |
| 166 | headers: { Accept: "application/json", "Content-Type": "application/json" }, |
| 167 | body: JSON.stringify({ |
| 168 | client_id: config.clientId, |
| 169 | client_secret: config.clientSecret, |
| 170 | ...body, |
| 171 | }), |
| 172 | }); |
| 173 | } catch (error) { |
| 174 | if (error.name === "TimeoutError" || error.name === "AbortError") { |
| 175 | throw new Error("GitHub did not respond in time. Start the sign-in again."); |
| 176 | } |
| 177 | throw error; |
| 178 | } |
| 179 | const data = await response.json(); |
| 180 | if (!response.ok || data.error || !data.access_token) { |
| 181 | throw new Error(data.error_description || "GitHub sign-in failed"); |
| 182 | } |
| 183 | return { |
| 184 | accessToken: data.access_token, |
| 185 | refreshToken: data.refresh_token || null, |
| 186 | expiresAt: data.expires_in |
| 187 | ? new Date(Date.now() + data.expires_in * 1000).toISOString() |
| 188 | : null, |
| 189 | }; |
| 190 | } |
| 191 | |
| 192 | return { |
| 193 | authorizationUrl(state) { |
| 194 | const params = new URLSearchParams({ |
| 195 | client_id: config.clientId, |
| 196 | redirect_uri: `${config.baseUrl}/auth/github/callback`, |
| 197 | state, |
| 198 | }); |
| 199 | return `https://github.com/login/oauth/authorize?${params}`; |
| 200 | }, |
| 201 | installationUrl(state) { |
| 202 | return `https://github.com/apps/${encodeURIComponent(config.appSlug)}/installations/new?state=${encodeURIComponent(state)}`; |
| 203 | }, |
| 204 | async exchangeCode(code) { |
| 205 | return requestUserToken({ code }); |
| 206 | }, |
| 207 | refreshUserToken(refreshToken) { |
| 208 | return requestUserToken({ |
| 209 | grant_type: "refresh_token", |
| 210 | refresh_token: refreshToken, |
| 211 | }); |
| 212 | }, |
| 213 | getViewer(token) { |
| 214 | return request("/user", token); |
| 215 | }, |
| 216 | async listRepositories(token, installationId) { |
| 217 | const repos = await allPages(`/user/installations/${installationId}/repositories`, token); |
| 218 | return repos.map((repo) => ({ |
| 219 | id: repo.id, |
| 220 | name: repo.name, |
| 221 | fullName: repo.full_name, |
| 222 | description: repo.description, |
| 223 | private: repo.private, |
| 224 | language: repo.language, |
| 225 | updatedAt: repo.updated_at, |
| 226 | defaultBranch: repo.default_branch, |
| 227 | owner: repo.owner.login, |
| 228 | })); |
| 229 | }, |
| 230 | async snapshotRepositories(token, selectedRepos) { |
| 231 | return mapLimit(selectedRepos, 2, async (selected) => { |
| 232 | const root = `/repos/${selected.fullName}`; |
| 233 | const repo = await request(root, token); |
| 234 | if (!repo.default_branch) { |
| 235 | return { |
| 236 | id: repo.id, |
| 237 | name: repo.name, |
| 238 | fullName: repo.full_name, |
| 239 | description: repo.description, |
| 240 | private: repo.private, |
| 241 | language: repo.language, |
| 242 | updatedAt: repo.updated_at, |
| 243 | createdAt: repo.created_at, |
| 244 | homepage: repo.homepage || null, |
| 245 | topics: repo.topics || [], |
| 246 | license: repo.license ? { name: repo.license.name, spdxId: repo.license.spdx_id } : null, |
| 247 | stargazersCount: repo.stargazers_count || 0, |
| 248 | forksCount: repo.forks_count || 0, |
| 249 | watchersCount: repo.subscribers_count || repo.watchers_count || 0, |
| 250 | archived: Boolean(repo.archived), |
| 251 | defaultBranch: null, |
| 252 | headSha: null, |
| 253 | branches: [], |
| 254 | tags: [], |
| 255 | refSnapshots: [], |
| 256 | issues: [], |
| 257 | pullRequests: [], |
| 258 | releases: [], |
| 259 | languages: [], |
| 260 | files: [], |
| 261 | commits: [], |
| 262 | }; |
| 263 | } |
| 264 | const [ |
| 265 | head, |
| 266 | branchRows, |
| 267 | tagRows, |
| 268 | issueRows, |
| 269 | pullRows, |
| 270 | releaseRows, |
| 271 | languageBytes, |
| 272 | ] = await Promise.all([ |
| 273 | request(`${root}/commits/${encodeURIComponent(repo.default_branch)}`, token), |
| 274 | allPages(`${root}/branches`, token), |
| 275 | allPages(`${root}/tags`, token), |
| 276 | optional(allPages(`${root}/issues?state=all&sort=updated&direction=desc`, token), []), |
| 277 | optional(allPages(`${root}/pulls?state=all&sort=updated&direction=desc`, token), []), |
| 278 | optional(allPages(`${root}/releases`, token), []), |
| 279 | request(`${root}/languages`, token), |
| 280 | ]); |
| 281 | |
| 282 | const branches = branchRows.map((branch) => ({ |
| 283 | name: branch.name, |
| 284 | sha: branch.commit.sha, |
| 285 | protected: Boolean(branch.protected), |
| 286 | })); |
| 287 | const defaultBranch = branches.find((branch) => branch.name === repo.default_branch); |
| 288 | if (defaultBranch) defaultBranch.sha = head.sha; |
| 289 | else branches.unshift({ name: repo.default_branch, sha: head.sha, protected: false }); |
| 290 | const tags = tagRows.map((tag) => ({ name: tag.name, sha: tag.commit.sha })); |
| 291 | const referenceShas = [...new Set([ |
| 292 | head.sha, |
| 293 | ...branches.map((branch) => branch.sha), |
| 294 | ...tags.map((tag) => tag.sha), |
| 295 | ])]; |
| 296 | const heads = await mapLimit(referenceShas, 4, (sha) => ( |
| 297 | sha === head.sha |
| 298 | ? head |
| 299 | : request(`${root}/commits/${encodeURIComponent(sha)}`, token) |
| 300 | )); |
| 301 | |
| 302 | const blobCache = new Map(); |
| 303 | async function snapshotFile(entry) { |
| 304 | if (entry.size > 500_000) { |
| 305 | return { path: entry.path, size: entry.size, content: null, truncated: true }; |
| 306 | } |
| 307 | let cached = blobCache.get(entry.sha); |
| 308 | if (!cached) { |
| 309 | cached = request(`${root}/git/blobs/${entry.sha}`, token); |
| 310 | blobCache.set(entry.sha, cached); |
| 311 | } |
| 312 | const blob = await cached; |
| 313 | const mediaType = imageMediaType(entry.path); |
| 314 | return { |
| 315 | path: entry.path, |
| 316 | size: entry.size, |
| 317 | content: !mediaType && blob.encoding === "base64" ? decodeFile(blob.content) : null, |
| 318 | binaryContent: mediaType && blob.encoding === "base64" |
| 319 | ? blob.content.replace(/\n/g, "") |
| 320 | : null, |
| 321 | mediaType, |
| 322 | truncated: false, |
| 323 | }; |
| 324 | } |
| 325 | |
| 326 | const commitCache = new Map(); |
| 327 | async function snapshotCommit(commit) { |
| 328 | let cached = commitCache.get(commit.sha); |
| 329 | if (!cached) { |
| 330 | cached = commitWithAllFiles(root, commit.sha, token).then((detail) => { |
| 331 | const metadata = commit.commit || detail.commit; |
| 332 | return { |
| 333 | sha: commit.sha, |
| 334 | message: metadata.message, |
| 335 | author: metadata.author.name, |
| 336 | date: metadata.author.date, |
| 337 | additions: detail.stats?.additions || 0, |
| 338 | deletions: detail.stats?.deletions || 0, |
| 339 | files: (detail.files || []).map((file) => ({ |
| 340 | filename: file.filename, |
| 341 | status: file.status, |
| 342 | additions: file.additions, |
| 343 | deletions: file.deletions, |
| 344 | patch: file.patch || "", |
| 345 | })), |
| 346 | }; |
| 347 | }); |
| 348 | commitCache.set(commit.sha, cached); |
| 349 | } |
| 350 | return cached; |
| 351 | } |
| 352 | |
| 353 | const refSnapshots = await mapLimit(heads, 2, async (refHead) => { |
| 354 | const [tree, commits] = await Promise.all([ |
| 355 | completeTree(root, refHead.commit.tree.sha, token), |
| 356 | allPages(`${root}/commits?sha=${encodeURIComponent(refHead.sha)}`, token), |
| 357 | ]); |
| 358 | return { |
| 359 | sha: refHead.sha, |
| 360 | files: await mapLimit(tree.filter((entry) => entry.type === "blob"), 8, snapshotFile), |
| 361 | commits: await mapLimit(commits, 4, snapshotCommit), |
| 362 | }; |
| 363 | }); |
| 364 | const issues = await mapLimit( |
| 365 | issueRows.filter((issue) => !issue.pull_request), |
| 366 | 4, |
| 367 | async (issue) => { |
| 368 | const comments = await allPages(`${root}/issues/${issue.number}/comments`, token); |
| 369 | return { |
| 370 | number: issue.number, |
| 371 | title: issue.title, |
| 372 | body: issue.body || "", |
| 373 | state: issue.state, |
| 374 | stateReason: issue.state_reason || null, |
| 375 | locked: Boolean(issue.locked), |
| 376 | author: account(issue.user), |
| 377 | labels: (issue.labels || []).map((label) => ({ |
| 378 | name: typeof label === "string" ? label : label.name, |
| 379 | color: typeof label === "string" ? null : label.color, |
| 380 | })), |
| 381 | createdAt: issue.created_at, |
| 382 | updatedAt: issue.updated_at, |
| 383 | closedAt: issue.closed_at, |
| 384 | comments: comments.map((comment) => conversationComment(comment)), |
| 385 | }; |
| 386 | }, |
| 387 | ); |
| 388 | const pullRequests = await mapLimit(pullRows, 3, async (pull) => { |
| 389 | const [detail, comments, reviews, reviewComments, files] = await Promise.all([ |
| 390 | request(`${root}/pulls/${pull.number}`, token), |
| 391 | allPages(`${root}/issues/${pull.number}/comments`, token), |
| 392 | allPages(`${root}/pulls/${pull.number}/reviews`, token), |
| 393 | allPages(`${root}/pulls/${pull.number}/comments`, token), |
| 394 | allPages(`${root}/pulls/${pull.number}/files`, token), |
| 395 | ]); |
| 396 | const conversation = [ |
| 397 | ...comments.map((comment) => conversationComment(comment)), |
| 398 | ...reviews.map((review) => conversationComment(review, "review")), |
| 399 | ...reviewComments.map((comment) => conversationComment(comment, "review-comment")), |
| 400 | ].sort((a, b) => new Date(a.createdAt) - new Date(b.createdAt)); |
| 401 | return { |
| 402 | number: detail.number, |
| 403 | title: detail.title, |
| 404 | body: detail.body || "", |
| 405 | state: detail.state, |
| 406 | draft: Boolean(detail.draft), |
| 407 | merged: Boolean(detail.merged_at), |
| 408 | mergeableState: detail.mergeable_state || null, |
| 409 | author: account(detail.user), |
| 410 | labels: (detail.labels || []).map((label) => ({ |
| 411 | name: typeof label === "string" ? label : label.name, |
| 412 | color: typeof label === "string" ? null : label.color, |
| 413 | })), |
| 414 | base: detail.base.ref, |
| 415 | head: detail.head.ref, |
| 416 | createdAt: detail.created_at, |
| 417 | updatedAt: detail.updated_at, |
| 418 | closedAt: detail.closed_at, |
| 419 | mergedAt: detail.merged_at, |
| 420 | additions: detail.additions || 0, |
| 421 | deletions: detail.deletions || 0, |
| 422 | changedFiles: detail.changed_files || files.length, |
| 423 | commitCount: detail.commits || 0, |
| 424 | conversation, |
| 425 | files: files.map((file) => ({ |
| 426 | filename: file.filename, |
| 427 | previousFilename: file.previous_filename || null, |
| 428 | status: file.status, |
| 429 | additions: file.additions, |
| 430 | deletions: file.deletions, |
| 431 | changes: file.changes, |
| 432 | patch: file.patch || "", |
| 433 | })), |
| 434 | }; |
| 435 | }); |
| 436 | const releases = releaseRows.map((release) => ({ |
| 437 | id: release.id, |
| 438 | tagName: release.tag_name, |
| 439 | targetCommitish: release.target_commitish, |
| 440 | name: release.name || release.tag_name, |
| 441 | body: release.body || "", |
| 442 | draft: Boolean(release.draft), |
| 443 | prerelease: Boolean(release.prerelease), |
| 444 | author: account(release.author), |
| 445 | createdAt: release.created_at, |
| 446 | publishedAt: release.published_at, |
| 447 | assets: (release.assets || []).map((asset) => ({ |
| 448 | id: asset.id, |
| 449 | name: asset.name, |
| 450 | label: asset.label || null, |
| 451 | size: asset.size, |
| 452 | downloadCount: asset.download_count, |
| 453 | contentType: asset.content_type, |
| 454 | })), |
| 455 | })); |
| 456 | const totalLanguageBytes = Object.values(languageBytes).reduce((total, bytes) => total + bytes, 0); |
| 457 | const languages = Object.entries(languageBytes) |
| 458 | .sort(([, left], [, right]) => right - left) |
| 459 | .map(([name, bytes]) => ({ |
| 460 | name, |
| 461 | bytes, |
| 462 | percent: totalLanguageBytes ? Number(((bytes / totalLanguageBytes) * 100).toFixed(1)) : 0, |
| 463 | })); |
| 464 | const defaultSnapshot = refSnapshots.find((snapshot) => snapshot.sha === head.sha); |
| 465 | return { |
| 466 | id: repo.id, |
| 467 | name: repo.name, |
| 468 | fullName: repo.full_name, |
| 469 | description: repo.description, |
| 470 | private: repo.private, |
| 471 | language: repo.language, |
| 472 | updatedAt: repo.updated_at, |
| 473 | createdAt: repo.created_at, |
| 474 | homepage: repo.homepage || null, |
| 475 | topics: repo.topics || [], |
| 476 | license: repo.license ? { name: repo.license.name, spdxId: repo.license.spdx_id } : null, |
| 477 | stargazersCount: repo.stargazers_count || 0, |
| 478 | forksCount: repo.forks_count || 0, |
| 479 | watchersCount: repo.subscribers_count || repo.watchers_count || 0, |
| 480 | archived: Boolean(repo.archived), |
| 481 | defaultBranch: repo.default_branch, |
| 482 | headSha: head.sha, |
| 483 | branches, |
| 484 | tags, |
| 485 | refSnapshots: refSnapshots.filter((snapshot) => snapshot.sha !== head.sha), |
| 486 | issues, |
| 487 | pullRequests, |
| 488 | releases, |
| 489 | languages, |
| 490 | files: defaultSnapshot.files, |
| 491 | commits: defaultSnapshot.commits, |
| 492 | }; |
| 493 | }); |
| 494 | }, |
| 495 | }; |
| 496 | } |
| 497 | |