From cb6b19618b62833a50da71d408614e42b56392f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 14:24:38 +0000 Subject: [PATCH 01/14] Add pagination for GitHub issues/PRs search API When fetching issues with per_page=100 and there are more results, now fetches all follow-up pages up to the GitHub Search API limit of 1000 results (10 pages). Uses total_count from API response to determine when all items have been fetched. --- src/hooks/useGitHubDataFetching.ts | 108 +++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index 54d53bf..6fd0af3 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -127,6 +127,81 @@ export const useGitHubDataFetching = ({ return allEvents; }; + // Fetch all search items (issues/PRs) for a username with pagination + const fetchAllSearchItems = async ( + username: string, + token: string, + startDate: string, + endDate: string, + onProgress: (message: string) => void + ): Promise => { + const allItems: GitHubItem[] = []; + let page = 1; + const perPage = GITHUB_API_PER_PAGE; + const maxPages = 10; // GitHub Search API allows up to 1000 results (10 pages * 100 per page) + + const searchQuery = `(author:${username} OR assignee:${username}) AND updated:${startDate}..${endDate} AND (is:issue OR is:pr)`; + + while (page <= maxPages) { + try { + onProgress(`Fetching issues/PRs page ${page} for ${username}...`); + + const response = await fetch( + `https://api.github.com/search/issues?q=${encodeURIComponent(searchQuery)}&per_page=${perPage}&page=${page}&sort=updated`, + { + headers: { + ...(token && { Authorization: `token ${token}` }), + Accept: 'application/vnd.github.v3+json', + }, + } + ); + + if (!response.ok) { + const responseJSON = await response.json(); + throw new Error(`Failed to fetch issues/PRs page ${page}: ${responseJSON.message}`); + } + + const searchData = await response.json(); + const totalCount = searchData.total_count; + const items = searchData.items; + + // Add original property to search items and ensure assignee data is preserved + const itemsWithOriginal = items.map((item: Record) => ({ + ...item, + assignee: item.assignee || null, + assignees: item.assignees || [], + original: item, + })); + + allItems.push(...itemsWithOriginal); + + // Check if we've fetched all items + if (allItems.length >= totalCount || items.length < perPage) { + break; + } + + page++; + + // Add a small delay to respect GitHub API rate limits + await new Promise(resolve => setTimeout(resolve, GITHUB_API_DELAY_MS)); + + } catch (error) { + console.error(`Error fetching issues/PRs page ${page} for ${username}:`, error); + throw error; + } + } + + // Log if we hit the pagination limit + if (page > maxPages) { + console.warn( + `Reached GitHub Search API pagination limit (${maxPages} pages) for ${username}. ` + + `Returning ${allItems.length} items. GitHub API limits search results to 1000 items.` + ); + } + + return allItems; + }; + const handleSearch = useCallback(async () => { if (!username.trim()) { onError('Please enter a GitHub username'); @@ -208,35 +283,10 @@ export const useGitHubDataFetching = ({ setCurrentUsername(singleUsername); try { - - // Fetch issues and PRs with date range filtering - const searchQuery = `(author:${singleUsername} OR assignee:${singleUsername}) AND updated:${startDate}..${endDate} AND (is:issue OR is:pr)`; - const searchResponse = await fetch( - `https://api.github.com/search/issues?q=${encodeURIComponent(searchQuery)}&per_page=${GITHUB_API_PER_PAGE}&sort=updated&advanced_search=true`, - { - headers: { - ...(githubToken && { Authorization: `token ${githubToken}` }), - Accept: 'application/vnd.github.v3+json', - }, - } - ); - - if (!searchResponse.ok) { - const responseJSON = await searchResponse.json(); - throw new Error(`Failed to fetch issues/PRs: ${responseJSON.message}`); - } - - const searchData = await searchResponse.json(); - // Add original property to search items and ensure assignee data is preserved - const searchItemsWithOriginal = searchData.items.map((item: Record) => ({ - ...item, - // Ensure assignee information is preserved from the API response - assignee: item.assignee || null, - assignees: item.assignees || [], - original: item, // Store the original item as the original payload - })); - allSearchItems.push(...searchItemsWithOriginal); - onProgress(`Fetched issues/PRs for ${singleUsername}`); + // Fetch all issues and PRs with pagination + const userSearchItems = await fetchAllSearchItems(singleUsername, githubToken, startDate, endDate, onProgress); + allSearchItems.push(...userSearchItems); + onProgress(`Fetched ${userSearchItems.length} issues/PRs for ${singleUsername}`); // Fetch all events with pagination const userEvents = await fetchAllEvents(singleUsername, githubToken, startDate, endDate, onProgress); From 433d5cf04f0477b1f530fb9a666c7fa7350da548 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 14:52:42 +0000 Subject: [PATCH 02/14] Initial plan From cad6b5049298f910da90bfb98221d7f95abf47f9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 14:55:33 +0000 Subject: [PATCH 03/14] Fix pagination limit warning condition - Changed condition from `page > maxPages` (which would never be true) to `allItems.length < totalCount` - Moved totalCount variable outside the loop to be accessible after loop completion - Now properly detects when pagination limit is hit with more items remaining - Updated warning message to show actual vs total count Co-authored-by: kertal <463851+kertal@users.noreply.github.com> --- src/hooks/useGitHubDataFetching.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index 6fd0af3..c06f10e 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -139,6 +139,7 @@ export const useGitHubDataFetching = ({ let page = 1; const perPage = GITHUB_API_PER_PAGE; const maxPages = 10; // GitHub Search API allows up to 1000 results (10 pages * 100 per page) + let totalCount = 0; const searchQuery = `(author:${username} OR assignee:${username}) AND updated:${startDate}..${endDate} AND (is:issue OR is:pr)`; @@ -162,7 +163,7 @@ export const useGitHubDataFetching = ({ } const searchData = await response.json(); - const totalCount = searchData.total_count; + totalCount = searchData.total_count; const items = searchData.items; // Add original property to search items and ensure assignee data is preserved @@ -191,11 +192,11 @@ export const useGitHubDataFetching = ({ } } - // Log if we hit the pagination limit - if (page > maxPages) { + // Log if we hit the pagination limit but there are more items available + if (allItems.length < totalCount) { console.warn( `Reached GitHub Search API pagination limit (${maxPages} pages) for ${username}. ` + - `Returning ${allItems.length} items. GitHub API limits search results to 1000 items.` + `Returning ${allItems.length} items out of ${totalCount} total. GitHub API limits search results to 1000 items.` ); } From b53bb999445cc637e226f4d3eff4e7751a2a3077 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 20:18:31 +0000 Subject: [PATCH 04/14] Add tests for GitHub issues/PRs search pagination Tests cover: - Fetching multiple pages when total_count exceeds per_page - Stopping when all items are retrieved (items.length < perPage) - Respecting max pages limit (10 pages / 1000 results) - Preserving assignee data and original property on items --- src/hooks/useGitHubDataFetching.test.ts | 182 +++++++++++++++++++++++- 1 file changed, 181 insertions(+), 1 deletion(-) diff --git a/src/hooks/useGitHubDataFetching.test.ts b/src/hooks/useGitHubDataFetching.test.ts index 44c902e..1115964 100644 --- a/src/hooks/useGitHubDataFetching.test.ts +++ b/src/hooks/useGitHubDataFetching.test.ts @@ -249,8 +249,188 @@ describe('useGitHubDataFetching', () => { currentUsername: '', handleSearch: expect.any(Function), }); - + // The implementation should stop at page 3 even if more pages are available // This is handled internally by the maxPages constant in fetchAllEvents }); + + describe('search items pagination', () => { + const createMockSearchResponse = (items: object[], totalCount: number) => ({ + total_count: totalCount, + incomplete_results: false, + items: items, + }); + + const createMockItem = (id: number) => ({ + id, + number: id, + title: `Issue ${id}`, + html_url: `https://github.com/test/repo/issues/${id}`, + state: 'open', + created_at: '2024-01-15T10:00:00Z', + updated_at: '2024-01-15T10:00:00Z', + user: { login: 'testuser', id: 1 }, + }); + + const createMockEventsResponse = () => []; + + beforeEach(() => { + vi.clearAllMocks(); + (global.fetch as ReturnType).mockReset(); + }); + + it('should fetch multiple pages when total_count exceeds per_page', async () => { + // Create 150 items total - should require 2 pages + const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); + const page2Items = Array.from({ length: 50 }, (_, i) => createMockItem(i + 101)); + + let searchCallCount = 0; + (global.fetch as ReturnType).mockImplementation((url: string) => { + if (url.includes('/search/issues')) { + searchCallCount++; + if (searchCallCount === 1) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(page1Items, 150)), + }); + } else { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(page2Items, 150)), + }); + } + } + // Events API - return empty + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockEventsResponse()), + }); + }); + + const { result } = renderHook(() => useGitHubDataFetching(defaultProps)); + + // Start the search + const searchPromise = result.current.handleSearch(); + + // Advance timers to allow async operations + await vi.runAllTimersAsync(); + await searchPromise; + + // Verify search API was called twice (2 pages) + const searchCalls = (global.fetch as ReturnType).mock.calls.filter( + (call: string[]) => call[0].includes('/search/issues') + ); + expect(searchCalls.length).toBe(2); + + // Verify page parameter increments + expect(searchCalls[0][0]).toContain('page=1'); + expect(searchCalls[1][0]).toContain('page=2'); + + // Verify storeSearchItems was called with all 150 items + expect(mockStoreSearchItems).toHaveBeenCalled(); + const storedItems = mockStoreSearchItems.mock.calls[0][1]; + expect(storedItems.length).toBe(150); + }); + + it('should stop fetching when all items are retrieved (items.length < perPage)', async () => { + // Only 50 items total - single page + const items = Array.from({ length: 50 }, (_, i) => createMockItem(i + 1)); + + (global.fetch as ReturnType).mockImplementation((url: string) => { + if (url.includes('/search/issues')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(items, 50)), + }); + } + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockEventsResponse()), + }); + }); + + const { result } = renderHook(() => useGitHubDataFetching(defaultProps)); + + const searchPromise = result.current.handleSearch(); + await vi.runAllTimersAsync(); + await searchPromise; + + // Should only call search API once since items < 100 + const searchCalls = (global.fetch as ReturnType).mock.calls.filter( + (call: string[]) => call[0].includes('/search/issues') + ); + expect(searchCalls.length).toBe(1); + }); + + it('should stop at max pages limit (10 pages)', async () => { + // Simulate a response that always returns 100 items with huge total_count + const pageItems = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); + + (global.fetch as ReturnType).mockImplementation((url: string) => { + if (url.includes('/search/issues')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(pageItems, 5000)), // More than 1000 + }); + } + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockEventsResponse()), + }); + }); + + const { result } = renderHook(() => useGitHubDataFetching(defaultProps)); + + const searchPromise = result.current.handleSearch(); + await vi.runAllTimersAsync(); + await searchPromise; + + // Should stop at 10 pages max + const searchCalls = (global.fetch as ReturnType).mock.calls.filter( + (call: string[]) => call[0].includes('/search/issues') + ); + expect(searchCalls.length).toBe(10); + }); + + it('should preserve assignee data and add original property', async () => { + const itemWithAssignee = { + id: 1, + number: 1, + title: 'Test Issue', + html_url: 'https://github.com/test/repo/issues/1', + state: 'open', + created_at: '2024-01-15T10:00:00Z', + updated_at: '2024-01-15T10:00:00Z', + user: { login: 'testuser', id: 1 }, + assignee: { login: 'assignee1', id: 2 }, + assignees: [{ login: 'assignee1', id: 2 }, { login: 'assignee2', id: 3 }], + }; + + (global.fetch as ReturnType).mockImplementation((url: string) => { + if (url.includes('/search/issues')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse([itemWithAssignee], 1)), + }); + } + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockEventsResponse()), + }); + }); + + const { result } = renderHook(() => useGitHubDataFetching(defaultProps)); + + const searchPromise = result.current.handleSearch(); + await vi.runAllTimersAsync(); + await searchPromise; + + expect(mockStoreSearchItems).toHaveBeenCalled(); + const storedItems = mockStoreSearchItems.mock.calls[0][1]; + expect(storedItems.length).toBe(1); + expect(storedItems[0].assignee).toEqual({ login: 'assignee1', id: 2 }); + expect(storedItems[0].assignees).toEqual([{ login: 'assignee1', id: 2 }, { login: 'assignee2', id: 3 }]); + expect(storedItems[0].original).toBeDefined(); + }); + }); }); \ No newline at end of file From 0df989bcc207a4452c377ba947deeddd8375c835 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 21:51:48 +0000 Subject: [PATCH 05/14] Improve error resilience in search pagination - Add 422 pagination limit error handling (consistent with fetchAllEvents) - Return partial results on error instead of throwing when items exist - Add tests for 422 handling and partial results on network error --- src/hooks/useGitHubDataFetching.test.ts | 76 +++++++++++++++++++++++++ src/hooks/useGitHubDataFetching.ts | 14 +++++ 2 files changed, 90 insertions(+) diff --git a/src/hooks/useGitHubDataFetching.test.ts b/src/hooks/useGitHubDataFetching.test.ts index 1115964..f5b7a67 100644 --- a/src/hooks/useGitHubDataFetching.test.ts +++ b/src/hooks/useGitHubDataFetching.test.ts @@ -432,5 +432,81 @@ describe('useGitHubDataFetching', () => { expect(storedItems[0].assignees).toEqual([{ login: 'assignee1', id: 2 }, { login: 'assignee2', id: 3 }]); expect(storedItems[0].original).toBeDefined(); }); + + it('should handle 422 pagination limit error gracefully', async () => { + // First page succeeds, second page returns 422 + const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); + + let searchCallCount = 0; + (global.fetch as ReturnType).mockImplementation((url: string) => { + if (url.includes('/search/issues')) { + searchCallCount++; + if (searchCallCount === 1) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), + }); + } else { + // Return 422 pagination limit error + return Promise.resolve({ + ok: false, + status: 422, + json: () => Promise.resolve({ message: 'Only the first 1000 search results are available. pagination is limited.' }), + }); + } + } + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockEventsResponse()), + }); + }); + + const { result } = renderHook(() => useGitHubDataFetching(defaultProps)); + + const searchPromise = result.current.handleSearch(); + await vi.runAllTimersAsync(); + await searchPromise; + + // Should store the 100 items from page 1 + expect(mockStoreSearchItems).toHaveBeenCalled(); + const storedItems = mockStoreSearchItems.mock.calls[0][1]; + expect(storedItems.length).toBe(100); + }); + + it('should return partial results when error occurs mid-pagination', async () => { + // First page succeeds, second page throws network error + const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); + + let searchCallCount = 0; + (global.fetch as ReturnType).mockImplementation((url: string) => { + if (url.includes('/search/issues')) { + searchCallCount++; + if (searchCallCount === 1) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), + }); + } else { + // Simulate network error + return Promise.reject(new Error('Network error')); + } + } + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockEventsResponse()), + }); + }); + + const { result } = renderHook(() => useGitHubDataFetching(defaultProps)); + + const searchPromise = result.current.handleSearch(); + await vi.runAllTimersAsync(); + await searchPromise; + + // Should store the 100 items from page 1 (partial results) + expect(mockStoreSearchItems).toHaveBeenCalled(); + const storedItems = mockStoreSearchItems.mock.calls[0][1]; + expect(storedItems.length).toBe(100); + }); }); }); \ No newline at end of file diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index c06f10e..b03e54b 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -159,6 +159,15 @@ export const useGitHubDataFetching = ({ if (!response.ok) { const responseJSON = await response.json(); + + // Handle pagination limit error (422) - return what we have so far + if (response.status === 422 && responseJSON.message?.includes('pagination')) { + console.warn( + `GitHub Search API pagination limit reached for ${username} at page ${page}. Returning ${allItems.length} items collected so far.` + ); + break; + } + throw new Error(`Failed to fetch issues/PRs page ${page}: ${responseJSON.message}`); } @@ -188,6 +197,11 @@ export const useGitHubDataFetching = ({ } catch (error) { console.error(`Error fetching issues/PRs page ${page} for ${username}:`, error); + // Return partial results if we have any, otherwise throw + if (allItems.length > 0) { + console.warn(`Returning ${allItems.length} items collected before error`); + return allItems; + } throw error; } } From 1e509b050ccd438b7162d5ef01d0cdc867ce4752 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 22:03:13 +0000 Subject: [PATCH 06/14] Fix GitHub search query syntax: use is:pull-request instead of is:pr --- src/hooks/useGitHubDataFetching.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index b03e54b..b3e6837 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -141,7 +141,7 @@ export const useGitHubDataFetching = ({ const maxPages = 10; // GitHub Search API allows up to 1000 results (10 pages * 100 per page) let totalCount = 0; - const searchQuery = `(author:${username} OR assignee:${username}) AND updated:${startDate}..${endDate} AND (is:issue OR is:pr)`; + const searchQuery = `(author:${username} OR assignee:${username}) updated:${startDate}..${endDate} (is:issue OR is:pull-request)`; while (page <= maxPages) { try { From decfafe8a1fbdd6b6cdf40f3d9ec4a3cbbe0be86 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 22:15:23 +0000 Subject: [PATCH 07/14] Split search queries for issues and PRs separately GitHub Apps with user access tokens cannot combine is:issue and is:pull-request in a single query. Now makes separate API calls for issues and PRs, then combines the results. Updated tests to account for the separate queries. --- src/hooks/useGitHubDataFetching.test.ts | 140 +++++++++++++++--------- src/hooks/useGitHubDataFetching.ts | 124 ++++++++++----------- 2 files changed, 150 insertions(+), 114 deletions(-) diff --git a/src/hooks/useGitHubDataFetching.test.ts b/src/hooks/useGitHubDataFetching.test.ts index f5b7a67..a58f107 100644 --- a/src/hooks/useGitHubDataFetching.test.ts +++ b/src/hooks/useGitHubDataFetching.test.ts @@ -280,25 +280,33 @@ describe('useGitHubDataFetching', () => { }); it('should fetch multiple pages when total_count exceeds per_page', async () => { - // Create 150 items total - should require 2 pages + // Create 150 issues total - should require 2 pages for issues const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); const page2Items = Array.from({ length: 50 }, (_, i) => createMockItem(i + 101)); - let searchCallCount = 0; + let issueCallCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - searchCallCount++; - if (searchCallCount === 1) { - return Promise.resolve({ - ok: true, - json: () => Promise.resolve(createMockSearchResponse(page1Items, 150)), - }); - } else { - return Promise.resolve({ - ok: true, - json: () => Promise.resolve(createMockSearchResponse(page2Items, 150)), - }); + // Check if this is an issue or PR query + if (url.includes('is%3Aissue')) { + issueCallCount++; + if (issueCallCount === 1) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(page1Items, 150)), + }); + } else { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(page2Items, 150)), + }); + } } + // PR query - return empty + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse([], 0)), + }); } // Events API - return empty return Promise.resolve({ @@ -316,15 +324,15 @@ describe('useGitHubDataFetching', () => { await vi.runAllTimersAsync(); await searchPromise; - // Verify search API was called twice (2 pages) - const searchCalls = (global.fetch as ReturnType).mock.calls.filter( - (call: string[]) => call[0].includes('/search/issues') + // Verify search API was called for issues (2 pages) + PRs (1 page) + const issueCalls = (global.fetch as ReturnType).mock.calls.filter( + (call: string[]) => call[0].includes('/search/issues') && call[0].includes('is%3Aissue') ); - expect(searchCalls.length).toBe(2); + expect(issueCalls.length).toBe(2); - // Verify page parameter increments - expect(searchCalls[0][0]).toContain('page=1'); - expect(searchCalls[1][0]).toContain('page=2'); + // Verify page parameter increments for issues + expect(issueCalls[0][0]).toContain('page=1'); + expect(issueCalls[1][0]).toContain('page=2'); // Verify storeSearchItems was called with all 150 items expect(mockStoreSearchItems).toHaveBeenCalled(); @@ -333,14 +341,21 @@ describe('useGitHubDataFetching', () => { }); it('should stop fetching when all items are retrieved (items.length < perPage)', async () => { - // Only 50 items total - single page + // Only 50 issues total - single page per type const items = Array.from({ length: 50 }, (_, i) => createMockItem(i + 1)); (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { + if (url.includes('is%3Aissue')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(items, 50)), + }); + } + // PR query - return empty return Promise.resolve({ ok: true, - json: () => Promise.resolve(createMockSearchResponse(items, 50)), + json: () => Promise.resolve(createMockSearchResponse([], 0)), }); } return Promise.resolve({ @@ -355,14 +370,14 @@ describe('useGitHubDataFetching', () => { await vi.runAllTimersAsync(); await searchPromise; - // Should only call search API once since items < 100 + // Should call search API twice (once for issues, once for PRs) const searchCalls = (global.fetch as ReturnType).mock.calls.filter( (call: string[]) => call[0].includes('/search/issues') ); - expect(searchCalls.length).toBe(1); + expect(searchCalls.length).toBe(2); // 1 for issues + 1 for PRs }); - it('should stop at max pages limit (10 pages)', async () => { + it('should stop at max pages limit (10 pages per type)', async () => { // Simulate a response that always returns 100 items with huge total_count const pageItems = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); @@ -385,11 +400,11 @@ describe('useGitHubDataFetching', () => { await vi.runAllTimersAsync(); await searchPromise; - // Should stop at 10 pages max + // Should stop at 10 pages max per type (10 for issues + 10 for PRs = 20) const searchCalls = (global.fetch as ReturnType).mock.calls.filter( (call: string[]) => call[0].includes('/search/issues') ); - expect(searchCalls.length).toBe(10); + expect(searchCalls.length).toBe(20); }); it('should preserve assignee data and add original property', async () => { @@ -408,9 +423,16 @@ describe('useGitHubDataFetching', () => { (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { + if (url.includes('is%3Aissue')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse([itemWithAssignee], 1)), + }); + } + // PR query - return empty return Promise.resolve({ ok: true, - json: () => Promise.resolve(createMockSearchResponse([itemWithAssignee], 1)), + json: () => Promise.resolve(createMockSearchResponse([], 0)), }); } return Promise.resolve({ @@ -437,23 +459,30 @@ describe('useGitHubDataFetching', () => { // First page succeeds, second page returns 422 const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); - let searchCallCount = 0; + let issueCallCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - searchCallCount++; - if (searchCallCount === 1) { - return Promise.resolve({ - ok: true, - json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), - }); - } else { - // Return 422 pagination limit error - return Promise.resolve({ - ok: false, - status: 422, - json: () => Promise.resolve({ message: 'Only the first 1000 search results are available. pagination is limited.' }), - }); + if (url.includes('is%3Aissue')) { + issueCallCount++; + if (issueCallCount === 1) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), + }); + } else { + // Return 422 pagination limit error + return Promise.resolve({ + ok: false, + status: 422, + json: () => Promise.resolve({ message: 'Only the first 1000 search results are available. pagination is limited.' }), + }); + } } + // PR query - return empty + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse([], 0)), + }); } return Promise.resolve({ ok: true, @@ -477,19 +506,26 @@ describe('useGitHubDataFetching', () => { // First page succeeds, second page throws network error const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); - let searchCallCount = 0; + let issueCallCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - searchCallCount++; - if (searchCallCount === 1) { - return Promise.resolve({ - ok: true, - json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), - }); - } else { - // Simulate network error - return Promise.reject(new Error('Network error')); + if (url.includes('is%3Aissue')) { + issueCallCount++; + if (issueCallCount === 1) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), + }); + } else { + // Simulate network error + return Promise.reject(new Error('Network error')); + } } + // PR query - return empty + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse([], 0)), + }); } return Promise.resolve({ ok: true, diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index b3e6837..935b490 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -136,84 +136,84 @@ export const useGitHubDataFetching = ({ onProgress: (message: string) => void ): Promise => { const allItems: GitHubItem[] = []; - let page = 1; const perPage = GITHUB_API_PER_PAGE; const maxPages = 10; // GitHub Search API allows up to 1000 results (10 pages * 100 per page) - let totalCount = 0; - - const searchQuery = `(author:${username} OR assignee:${username}) updated:${startDate}..${endDate} (is:issue OR is:pull-request)`; - while (page <= maxPages) { - try { - onProgress(`Fetching issues/PRs page ${page} for ${username}...`); + // GitHub Apps with user access tokens require separate queries for issues and PRs + const types = ['issue', 'pull-request'] as const; - const response = await fetch( - `https://api.github.com/search/issues?q=${encodeURIComponent(searchQuery)}&per_page=${perPage}&page=${page}&sort=updated`, - { - headers: { - ...(token && { Authorization: `token ${token}` }), - Accept: 'application/vnd.github.v3+json', - }, - } - ); - - if (!response.ok) { - const responseJSON = await response.json(); - - // Handle pagination limit error (422) - return what we have so far - if (response.status === 422 && responseJSON.message?.includes('pagination')) { - console.warn( - `GitHub Search API pagination limit reached for ${username} at page ${page}. Returning ${allItems.length} items collected so far.` - ); - break; - } + for (const type of types) { + let page = 1; + let totalCount = 0; + const searchQuery = `(author:${username} OR assignee:${username}) updated:${startDate}..${endDate} is:${type}`; - throw new Error(`Failed to fetch issues/PRs page ${page}: ${responseJSON.message}`); - } + while (page <= maxPages) { + try { + onProgress(`Fetching ${type === 'pull-request' ? 'PRs' : 'issues'} page ${page} for ${username}...`); + + const response = await fetch( + `https://api.github.com/search/issues?q=${encodeURIComponent(searchQuery)}&per_page=${perPage}&page=${page}&sort=updated`, + { + headers: { + ...(token && { Authorization: `token ${token}` }), + Accept: 'application/vnd.github.v3+json', + }, + } + ); - const searchData = await response.json(); - totalCount = searchData.total_count; - const items = searchData.items; + if (!response.ok) { + const responseJSON = await response.json(); - // Add original property to search items and ensure assignee data is preserved - const itemsWithOriginal = items.map((item: Record) => ({ - ...item, - assignee: item.assignee || null, - assignees: item.assignees || [], - original: item, - })); + // Handle pagination limit error (422) - continue with next type + if (response.status === 422 && responseJSON.message?.includes('pagination')) { + console.warn( + `GitHub Search API pagination limit reached for ${username} ${type} at page ${page}.` + ); + break; + } - allItems.push(...itemsWithOriginal); + throw new Error(`Failed to fetch ${type} page ${page}: ${responseJSON.message}`); + } - // Check if we've fetched all items - if (allItems.length >= totalCount || items.length < perPage) { - break; - } + const searchData = await response.json(); + totalCount = searchData.total_count; + const items = searchData.items; + + // Add original property to search items and ensure assignee data is preserved + const itemsWithOriginal = items.map((item: Record) => ({ + ...item, + assignee: item.assignee || null, + assignees: item.assignees || [], + original: item, + })); + + allItems.push(...itemsWithOriginal); + + // Check if we've fetched all items for this type + const itemsForType = allItems.filter((item: GitHubItem) => + type === 'pull-request' ? item.pull_request : !item.pull_request + ).length; + if (itemsForType >= totalCount || items.length < perPage) { + break; + } - page++; + page++; - // Add a small delay to respect GitHub API rate limits - await new Promise(resolve => setTimeout(resolve, GITHUB_API_DELAY_MS)); + // Add a small delay to respect GitHub API rate limits + await new Promise(resolve => setTimeout(resolve, GITHUB_API_DELAY_MS)); - } catch (error) { - console.error(`Error fetching issues/PRs page ${page} for ${username}:`, error); - // Return partial results if we have any, otherwise throw - if (allItems.length > 0) { - console.warn(`Returning ${allItems.length} items collected before error`); - return allItems; + } catch (error) { + console.error(`Error fetching ${type} page ${page} for ${username}:`, error); + // Return partial results if we have any, otherwise throw + if (allItems.length > 0) { + console.warn(`Returning ${allItems.length} items collected before error`); + return allItems; + } + throw error; } - throw error; } } - // Log if we hit the pagination limit but there are more items available - if (allItems.length < totalCount) { - console.warn( - `Reached GitHub Search API pagination limit (${maxPages} pages) for ${username}. ` + - `Returning ${allItems.length} items out of ${totalCount} total. GitHub API limits search results to 1000 items.` - ); - } - return allItems; }; From 8fac88cb4c9779992fee9634db84ba9612437dd8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 22:23:08 +0000 Subject: [PATCH 08/14] Remove parentheses from GitHub search queries GitHub search API doesn't support parentheses for grouping. Now makes 4 separate queries per user: - author:username is:issue - assignee:username is:issue - author:username is:pull-request - assignee:username is:pull-request Results are deduplicated by id to handle items matching multiple queries. Added test for deduplication. --- src/hooks/useGitHubDataFetching.test.ts | 110 ++++++++++++++++-------- src/hooks/useGitHubDataFetching.ts | 58 +++++++------ 2 files changed, 106 insertions(+), 62 deletions(-) diff --git a/src/hooks/useGitHubDataFetching.test.ts b/src/hooks/useGitHubDataFetching.test.ts index a58f107..5dbca42 100644 --- a/src/hooks/useGitHubDataFetching.test.ts +++ b/src/hooks/useGitHubDataFetching.test.ts @@ -280,17 +280,17 @@ describe('useGitHubDataFetching', () => { }); it('should fetch multiple pages when total_count exceeds per_page', async () => { - // Create 150 issues total - should require 2 pages for issues + // Create 150 issues total - should require 2 pages for author query const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); const page2Items = Array.from({ length: 50 }, (_, i) => createMockItem(i + 101)); - let issueCallCount = 0; + let authorIssueCallCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - // Check if this is an issue or PR query - if (url.includes('is%3Aissue')) { - issueCallCount++; - if (issueCallCount === 1) { + // Check if this is an issue author query (first query in the list) + if (url.includes('is%3Aissue') && url.includes('author%3A')) { + authorIssueCallCount++; + if (authorIssueCallCount === 1) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse(page1Items, 150)), @@ -302,7 +302,7 @@ describe('useGitHubDataFetching', () => { }); } } - // PR query - return empty + // Other queries (assignee, PRs) - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), @@ -324,15 +324,15 @@ describe('useGitHubDataFetching', () => { await vi.runAllTimersAsync(); await searchPromise; - // Verify search API was called for issues (2 pages) + PRs (1 page) - const issueCalls = (global.fetch as ReturnType).mock.calls.filter( - (call: string[]) => call[0].includes('/search/issues') && call[0].includes('is%3Aissue') + // Verify search API was called for issue author (2 pages) + const authorIssueCalls = (global.fetch as ReturnType).mock.calls.filter( + (call: string[]) => call[0].includes('/search/issues') && call[0].includes('is%3Aissue') && call[0].includes('author%3A') ); - expect(issueCalls.length).toBe(2); + expect(authorIssueCalls.length).toBe(2); // Verify page parameter increments for issues - expect(issueCalls[0][0]).toContain('page=1'); - expect(issueCalls[1][0]).toContain('page=2'); + expect(authorIssueCalls[0][0]).toContain('page=1'); + expect(authorIssueCalls[1][0]).toContain('page=2'); // Verify storeSearchItems was called with all 150 items expect(mockStoreSearchItems).toHaveBeenCalled(); @@ -341,18 +341,18 @@ describe('useGitHubDataFetching', () => { }); it('should stop fetching when all items are retrieved (items.length < perPage)', async () => { - // Only 50 issues total - single page per type + // Only 50 issues total - single page per query const items = Array.from({ length: 50 }, (_, i) => createMockItem(i + 1)); (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - if (url.includes('is%3Aissue')) { + if (url.includes('is%3Aissue') && url.includes('author%3A')) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse(items, 50)), }); } - // PR query - return empty + // Other queries - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), @@ -370,19 +370,21 @@ describe('useGitHubDataFetching', () => { await vi.runAllTimersAsync(); await searchPromise; - // Should call search API twice (once for issues, once for PRs) + // Should call search API 4 times (author+assignee for issues, author+assignee for PRs) const searchCalls = (global.fetch as ReturnType).mock.calls.filter( (call: string[]) => call[0].includes('/search/issues') ); - expect(searchCalls.length).toBe(2); // 1 for issues + 1 for PRs + expect(searchCalls.length).toBe(4); }); - it('should stop at max pages limit (10 pages per type)', async () => { - // Simulate a response that always returns 100 items with huge total_count - const pageItems = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); - + it('should stop at max pages limit (10 pages per query)', async () => { + // Simulate a response that always returns 100 unique items with huge total_count + let callCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { + callCount++; + // Each call returns unique items to avoid deduplication stopping pagination + const pageItems = Array.from({ length: 100 }, (_, i) => createMockItem(callCount * 1000 + i + 1)); return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse(pageItems, 5000)), // More than 1000 @@ -400,11 +402,11 @@ describe('useGitHubDataFetching', () => { await vi.runAllTimersAsync(); await searchPromise; - // Should stop at 10 pages max per type (10 for issues + 10 for PRs = 20) + // Should stop at 10 pages max per query (4 queries × 10 pages = 40) const searchCalls = (global.fetch as ReturnType).mock.calls.filter( (call: string[]) => call[0].includes('/search/issues') ); - expect(searchCalls.length).toBe(20); + expect(searchCalls.length).toBe(40); }); it('should preserve assignee data and add original property', async () => { @@ -423,13 +425,13 @@ describe('useGitHubDataFetching', () => { (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - if (url.includes('is%3Aissue')) { + if (url.includes('is%3Aissue') && url.includes('author%3A')) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([itemWithAssignee], 1)), }); } - // PR query - return empty + // Other queries - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), @@ -455,16 +457,52 @@ describe('useGitHubDataFetching', () => { expect(storedItems[0].original).toBeDefined(); }); + it('should deduplicate items returned by multiple queries', async () => { + // Same item returned by both author and assignee queries + const sharedItem = createMockItem(1); + + (global.fetch as ReturnType).mockImplementation((url: string) => { + if (url.includes('/search/issues')) { + if (url.includes('is%3Aissue')) { + // Both author and assignee queries return the same item + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse([sharedItem], 1)), + }); + } + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockSearchResponse([], 0)), + }); + } + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(createMockEventsResponse()), + }); + }); + + const { result } = renderHook(() => useGitHubDataFetching(defaultProps)); + + const searchPromise = result.current.handleSearch(); + await vi.runAllTimersAsync(); + await searchPromise; + + expect(mockStoreSearchItems).toHaveBeenCalled(); + const storedItems = mockStoreSearchItems.mock.calls[0][1]; + // Should only have 1 item even though it was returned by 2 queries + expect(storedItems.length).toBe(1); + }); + it('should handle 422 pagination limit error gracefully', async () => { // First page succeeds, second page returns 422 const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); - let issueCallCount = 0; + let authorIssueCallCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - if (url.includes('is%3Aissue')) { - issueCallCount++; - if (issueCallCount === 1) { + if (url.includes('is%3Aissue') && url.includes('author%3A')) { + authorIssueCallCount++; + if (authorIssueCallCount === 1) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), @@ -478,7 +516,7 @@ describe('useGitHubDataFetching', () => { }); } } - // PR query - return empty + // Other queries - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), @@ -506,12 +544,12 @@ describe('useGitHubDataFetching', () => { // First page succeeds, second page throws network error const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); - let issueCallCount = 0; + let authorIssueCallCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - if (url.includes('is%3Aissue')) { - issueCallCount++; - if (issueCallCount === 1) { + if (url.includes('is%3Aissue') && url.includes('author%3A')) { + authorIssueCallCount++; + if (authorIssueCallCount === 1) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), @@ -521,7 +559,7 @@ describe('useGitHubDataFetching', () => { return Promise.reject(new Error('Network error')); } } - // PR query - return empty + // Other queries - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index 935b490..e6253e5 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -136,20 +136,26 @@ export const useGitHubDataFetching = ({ onProgress: (message: string) => void ): Promise => { const allItems: GitHubItem[] = []; + const seenIds = new Set(); const perPage = GITHUB_API_PER_PAGE; const maxPages = 10; // GitHub Search API allows up to 1000 results (10 pages * 100 per page) // GitHub Apps with user access tokens require separate queries for issues and PRs - const types = ['issue', 'pull-request'] as const; - - for (const type of types) { + // Also need separate queries for author and assignee since parentheses aren't supported + const queries = [ + { type: 'issue', role: 'author', query: `author:${username} updated:${startDate}..${endDate} is:issue` }, + { type: 'issue', role: 'assignee', query: `assignee:${username} updated:${startDate}..${endDate} is:issue` }, + { type: 'pull-request', role: 'author', query: `author:${username} updated:${startDate}..${endDate} is:pull-request` }, + { type: 'pull-request', role: 'assignee', query: `assignee:${username} updated:${startDate}..${endDate} is:pull-request` }, + ]; + + for (const { type, role, query: searchQuery } of queries) { let page = 1; - let totalCount = 0; - const searchQuery = `(author:${username} OR assignee:${username}) updated:${startDate}..${endDate} is:${type}`; while (page <= maxPages) { try { - onProgress(`Fetching ${type === 'pull-request' ? 'PRs' : 'issues'} page ${page} for ${username}...`); + const typeLabel = type === 'pull-request' ? 'PRs' : 'issues'; + onProgress(`Fetching ${typeLabel} (${role}) page ${page} for ${username}...`); const response = await fetch( `https://api.github.com/search/issues?q=${encodeURIComponent(searchQuery)}&per_page=${perPage}&page=${page}&sort=updated`, @@ -164,36 +170,36 @@ export const useGitHubDataFetching = ({ if (!response.ok) { const responseJSON = await response.json(); - // Handle pagination limit error (422) - continue with next type + // Handle pagination limit error (422) - continue with next query if (response.status === 422 && responseJSON.message?.includes('pagination')) { console.warn( - `GitHub Search API pagination limit reached for ${username} ${type} at page ${page}.` + `GitHub Search API pagination limit reached for ${username} ${type} (${role}) at page ${page}.` ); break; } - throw new Error(`Failed to fetch ${type} page ${page}: ${responseJSON.message}`); + throw new Error(`Failed to fetch ${type} (${role}) page ${page}: ${responseJSON.message}`); } const searchData = await response.json(); - totalCount = searchData.total_count; + const totalCount = searchData.total_count; const items = searchData.items; - // Add original property to search items and ensure assignee data is preserved - const itemsWithOriginal = items.map((item: Record) => ({ - ...item, - assignee: item.assignee || null, - assignees: item.assignees || [], - original: item, - })); - - allItems.push(...itemsWithOriginal); - - // Check if we've fetched all items for this type - const itemsForType = allItems.filter((item: GitHubItem) => - type === 'pull-request' ? item.pull_request : !item.pull_request - ).length; - if (itemsForType >= totalCount || items.length < perPage) { + // Add items, deduplicating by id + for (const item of items) { + if (!seenIds.has(item.id)) { + seenIds.add(item.id); + allItems.push({ + ...item, + assignee: item.assignee || null, + assignees: item.assignees || [], + original: item, + }); + } + } + + // Check if we've fetched all items for this query + if (page * perPage >= totalCount || items.length < perPage) { break; } @@ -203,7 +209,7 @@ export const useGitHubDataFetching = ({ await new Promise(resolve => setTimeout(resolve, GITHUB_API_DELAY_MS)); } catch (error) { - console.error(`Error fetching ${type} page ${page} for ${username}:`, error); + console.error(`Error fetching ${type} (${role}) page ${page} for ${username}:`, error); // Return partial results if we have any, otherwise throw if (allItems.length > 0) { console.warn(`Returning ${allItems.length} items collected before error`); From e2be519a0dbd70a2fc2eff7bc72f64870124445b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 22:33:29 +0000 Subject: [PATCH 09/14] Initial plan From 8f0d52879bd0835bb74a5253f9a7f4b1b2ac4280 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 22:37:24 +0000 Subject: [PATCH 10/14] Extract maxPages to module constant and fix pagination termination logic Co-authored-by: kertal <463851+kertal@users.noreply.github.com> --- src/hooks/useGitHubDataFetching.ts | 12 ++++++++---- src/utils/settings.ts | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index e6253e5..b0c85ae 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -2,7 +2,7 @@ import { useCallback, useState } from 'react'; import { GitHubEvent, GitHubItem } from '../types'; import { EventsData } from '../utils/indexedDB'; import { validateUsernameList, isValidDateString } from '../utils'; -import { MAX_USERNAMES_PER_REQUEST, GITHUB_API_PER_PAGE, GITHUB_API_DELAY_MS } from '../utils/settings'; +import { MAX_USERNAMES_PER_REQUEST, GITHUB_API_PER_PAGE, GITHUB_API_DELAY_MS, GITHUB_SEARCH_API_MAX_PAGES } from '../utils/settings'; interface UseGitHubDataFetchingProps { username: string; @@ -138,7 +138,6 @@ export const useGitHubDataFetching = ({ const allItems: GitHubItem[] = []; const seenIds = new Set(); const perPage = GITHUB_API_PER_PAGE; - const maxPages = 10; // GitHub Search API allows up to 1000 results (10 pages * 100 per page) // GitHub Apps with user access tokens require separate queries for issues and PRs // Also need separate queries for author and assignee since parentheses aren't supported @@ -151,8 +150,9 @@ export const useGitHubDataFetching = ({ for (const { type, role, query: searchQuery } of queries) { let page = 1; + let itemsFetchedForQuery = 0; - while (page <= maxPages) { + while (page <= GITHUB_SEARCH_API_MAX_PAGES) { try { const typeLabel = type === 'pull-request' ? 'PRs' : 'issues'; onProgress(`Fetching ${typeLabel} (${role}) page ${page} for ${username}...`); @@ -185,6 +185,9 @@ export const useGitHubDataFetching = ({ const totalCount = searchData.total_count; const items = searchData.items; + // Track items fetched for this query (before deduplication) + itemsFetchedForQuery += items.length; + // Add items, deduplicating by id for (const item of items) { if (!seenIds.has(item.id)) { @@ -199,7 +202,8 @@ export const useGitHubDataFetching = ({ } // Check if we've fetched all items for this query - if (page * perPage >= totalCount || items.length < perPage) { + // Stop if we got fewer items than requested OR if we've fetched all available items for this query + if (items.length < perPage || itemsFetchedForQuery >= totalCount) { break; } diff --git a/src/utils/settings.ts b/src/utils/settings.ts index d66142e..9b53885 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -2,3 +2,4 @@ export const MAX_USERNAMES_PER_REQUEST = 15; export const GITHUB_API_PER_PAGE = 100; export const GITHUB_API_DELAY_MS = 100; +export const GITHUB_SEARCH_API_MAX_PAGES = 10; // GitHub Search API allows up to 1000 results (10 pages * 100 per page) From f095f65c93eb794e0a4c06b348411dba02b91877 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 22:48:08 +0000 Subject: [PATCH 11/14] Optimize queries using involves: qualifier Use "involves:username" instead of separate author/assignee queries. This reduces API calls from 4 to 2 per user (50% reduction). involves: matches author, assignee, commenter, and mentions in one query. --- src/hooks/useGitHubDataFetching.test.ts | 74 ++++++++++++------------- src/hooks/useGitHubDataFetching.ts | 18 +++--- 2 files changed, 42 insertions(+), 50 deletions(-) diff --git a/src/hooks/useGitHubDataFetching.test.ts b/src/hooks/useGitHubDataFetching.test.ts index 5dbca42..324711a 100644 --- a/src/hooks/useGitHubDataFetching.test.ts +++ b/src/hooks/useGitHubDataFetching.test.ts @@ -280,17 +280,17 @@ describe('useGitHubDataFetching', () => { }); it('should fetch multiple pages when total_count exceeds per_page', async () => { - // Create 150 issues total - should require 2 pages for author query + // Create 150 issues total - should require 2 pages const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); const page2Items = Array.from({ length: 50 }, (_, i) => createMockItem(i + 101)); - let authorIssueCallCount = 0; + let issueCallCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - // Check if this is an issue author query (first query in the list) - if (url.includes('is%3Aissue') && url.includes('author%3A')) { - authorIssueCallCount++; - if (authorIssueCallCount === 1) { + // Check if this is an issue query + if (url.includes('is%3Aissue')) { + issueCallCount++; + if (issueCallCount === 1) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse(page1Items, 150)), @@ -302,7 +302,7 @@ describe('useGitHubDataFetching', () => { }); } } - // Other queries (assignee, PRs) - return empty + // PR query - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), @@ -324,15 +324,15 @@ describe('useGitHubDataFetching', () => { await vi.runAllTimersAsync(); await searchPromise; - // Verify search API was called for issue author (2 pages) - const authorIssueCalls = (global.fetch as ReturnType).mock.calls.filter( - (call: string[]) => call[0].includes('/search/issues') && call[0].includes('is%3Aissue') && call[0].includes('author%3A') + // Verify search API was called for issues (2 pages) + const issueCalls = (global.fetch as ReturnType).mock.calls.filter( + (call: string[]) => call[0].includes('/search/issues') && call[0].includes('is%3Aissue') ); - expect(authorIssueCalls.length).toBe(2); + expect(issueCalls.length).toBe(2); // Verify page parameter increments for issues - expect(authorIssueCalls[0][0]).toContain('page=1'); - expect(authorIssueCalls[1][0]).toContain('page=2'); + expect(issueCalls[0][0]).toContain('page=1'); + expect(issueCalls[1][0]).toContain('page=2'); // Verify storeSearchItems was called with all 150 items expect(mockStoreSearchItems).toHaveBeenCalled(); @@ -346,13 +346,13 @@ describe('useGitHubDataFetching', () => { (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - if (url.includes('is%3Aissue') && url.includes('author%3A')) { + if (url.includes('is%3Aissue')) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse(items, 50)), }); } - // Other queries - return empty + // PR query - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), @@ -370,11 +370,11 @@ describe('useGitHubDataFetching', () => { await vi.runAllTimersAsync(); await searchPromise; - // Should call search API 4 times (author+assignee for issues, author+assignee for PRs) + // Should call search API 2 times (once for issues, once for PRs) const searchCalls = (global.fetch as ReturnType).mock.calls.filter( (call: string[]) => call[0].includes('/search/issues') ); - expect(searchCalls.length).toBe(4); + expect(searchCalls.length).toBe(2); }); it('should stop at max pages limit (10 pages per query)', async () => { @@ -402,11 +402,11 @@ describe('useGitHubDataFetching', () => { await vi.runAllTimersAsync(); await searchPromise; - // Should stop at 10 pages max per query (4 queries × 10 pages = 40) + // Should stop at 10 pages max per query (2 queries × 10 pages = 20) const searchCalls = (global.fetch as ReturnType).mock.calls.filter( (call: string[]) => call[0].includes('/search/issues') ); - expect(searchCalls.length).toBe(40); + expect(searchCalls.length).toBe(20); }); it('should preserve assignee data and add original property', async () => { @@ -425,13 +425,13 @@ describe('useGitHubDataFetching', () => { (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - if (url.includes('is%3Aissue') && url.includes('author%3A')) { + if (url.includes('is%3Aissue')) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([itemWithAssignee], 1)), }); } - // Other queries - return empty + // PR query - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), @@ -458,21 +458,15 @@ describe('useGitHubDataFetching', () => { }); it('should deduplicate items returned by multiple queries', async () => { - // Same item returned by both author and assignee queries + // Same item returned by both issue and PR queries (edge case) const sharedItem = createMockItem(1); (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - if (url.includes('is%3Aissue')) { - // Both author and assignee queries return the same item - return Promise.resolve({ - ok: true, - json: () => Promise.resolve(createMockSearchResponse([sharedItem], 1)), - }); - } + // Both issue and PR queries return the same item return Promise.resolve({ ok: true, - json: () => Promise.resolve(createMockSearchResponse([], 0)), + json: () => Promise.resolve(createMockSearchResponse([sharedItem], 1)), }); } return Promise.resolve({ @@ -497,12 +491,12 @@ describe('useGitHubDataFetching', () => { // First page succeeds, second page returns 422 const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); - let authorIssueCallCount = 0; + let issueCallCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - if (url.includes('is%3Aissue') && url.includes('author%3A')) { - authorIssueCallCount++; - if (authorIssueCallCount === 1) { + if (url.includes('is%3Aissue')) { + issueCallCount++; + if (issueCallCount === 1) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), @@ -516,7 +510,7 @@ describe('useGitHubDataFetching', () => { }); } } - // Other queries - return empty + // PR query - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), @@ -544,12 +538,12 @@ describe('useGitHubDataFetching', () => { // First page succeeds, second page throws network error const page1Items = Array.from({ length: 100 }, (_, i) => createMockItem(i + 1)); - let authorIssueCallCount = 0; + let issueCallCount = 0; (global.fetch as ReturnType).mockImplementation((url: string) => { if (url.includes('/search/issues')) { - if (url.includes('is%3Aissue') && url.includes('author%3A')) { - authorIssueCallCount++; - if (authorIssueCallCount === 1) { + if (url.includes('is%3Aissue')) { + issueCallCount++; + if (issueCallCount === 1) { return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse(page1Items, 200)), @@ -559,7 +553,7 @@ describe('useGitHubDataFetching', () => { return Promise.reject(new Error('Network error')); } } - // Other queries - return empty + // PR query - return empty return Promise.resolve({ ok: true, json: () => Promise.resolve(createMockSearchResponse([], 0)), diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index b0c85ae..2605078 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -140,22 +140,20 @@ export const useGitHubDataFetching = ({ const perPage = GITHUB_API_PER_PAGE; // GitHub Apps with user access tokens require separate queries for issues and PRs - // Also need separate queries for author and assignee since parentheses aren't supported + // Use "involves:" to match author, assignee, commenter, and mentions in one query const queries = [ - { type: 'issue', role: 'author', query: `author:${username} updated:${startDate}..${endDate} is:issue` }, - { type: 'issue', role: 'assignee', query: `assignee:${username} updated:${startDate}..${endDate} is:issue` }, - { type: 'pull-request', role: 'author', query: `author:${username} updated:${startDate}..${endDate} is:pull-request` }, - { type: 'pull-request', role: 'assignee', query: `assignee:${username} updated:${startDate}..${endDate} is:pull-request` }, + { type: 'issue', query: `involves:${username} updated:${startDate}..${endDate} is:issue` }, + { type: 'pull-request', query: `involves:${username} updated:${startDate}..${endDate} is:pull-request` }, ]; - for (const { type, role, query: searchQuery } of queries) { + for (const { type, query: searchQuery } of queries) { let page = 1; let itemsFetchedForQuery = 0; while (page <= GITHUB_SEARCH_API_MAX_PAGES) { try { const typeLabel = type === 'pull-request' ? 'PRs' : 'issues'; - onProgress(`Fetching ${typeLabel} (${role}) page ${page} for ${username}...`); + onProgress(`Fetching ${typeLabel} page ${page} for ${username}...`); const response = await fetch( `https://api.github.com/search/issues?q=${encodeURIComponent(searchQuery)}&per_page=${perPage}&page=${page}&sort=updated`, @@ -173,12 +171,12 @@ export const useGitHubDataFetching = ({ // Handle pagination limit error (422) - continue with next query if (response.status === 422 && responseJSON.message?.includes('pagination')) { console.warn( - `GitHub Search API pagination limit reached for ${username} ${type} (${role}) at page ${page}.` + `GitHub Search API pagination limit reached for ${username} ${type} at page ${page}.` ); break; } - throw new Error(`Failed to fetch ${type} (${role}) page ${page}: ${responseJSON.message}`); + throw new Error(`Failed to fetch ${type} page ${page}: ${responseJSON.message}`); } const searchData = await response.json(); @@ -213,7 +211,7 @@ export const useGitHubDataFetching = ({ await new Promise(resolve => setTimeout(resolve, GITHUB_API_DELAY_MS)); } catch (error) { - console.error(`Error fetching ${type} (${role}) page ${page} for ${username}:`, error); + console.error(`Error fetching ${type} page ${page} for ${username}:`, error); // Return partial results if we have any, otherwise throw if (allItems.length > 0) { console.warn(`Returning ${allItems.length} items collected before error`); From 5ba4aa992a7979e4d3a5872fb764f62f568d6356 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 23:09:32 +0000 Subject: [PATCH 12/14] Combine multiple users into single search queries Instead of making 2 API calls per user, now combines all users into just 2 total API calls (one for issues, one for PRs). Query format uses OR with repeated qualifiers: is:issue updated:... involves:user1 OR is:issue updated:... involves:user2 This works because AND has higher precedence than OR in GitHub search. For 5 users, this reduces API calls from 10 to 2 (80% reduction). Events API still requires per-user queries (no OR support). --- src/hooks/useGitHubDataFetching.ts | 65 ++++++++++++++++++------------ 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index 2605078..4c8c801 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -127,9 +127,10 @@ export const useGitHubDataFetching = ({ return allEvents; }; - // Fetch all search items (issues/PRs) for a username with pagination - const fetchAllSearchItems = async ( - username: string, + // Fetch all search items (issues/PRs) for multiple usernames with pagination + // Combines users into single queries to reduce API calls + const fetchAllSearchItemsForUsers = async ( + usernames: string[], token: string, startDate: string, endDate: string, @@ -139,11 +140,21 @@ export const useGitHubDataFetching = ({ const seenIds = new Set(); const perPage = GITHUB_API_PER_PAGE; - // GitHub Apps with user access tokens require separate queries for issues and PRs - // Use "involves:" to match author, assignee, commenter, and mentions in one query + // Build combined query for all users using OR + // Format: "is:issue updated:... involves:user1 OR is:issue updated:... involves:user2" + // This works because AND has higher precedence than OR in GitHub search + const buildCombinedQuery = (type: string) => { + const typeFilter = type === 'issue' ? 'is:issue' : 'is:pull-request'; + const dateFilter = `updated:${startDate}..${endDate}`; + + return usernames + .map(user => `${typeFilter} ${dateFilter} involves:${user}`) + .join(' OR '); + }; + const queries = [ - { type: 'issue', query: `involves:${username} updated:${startDate}..${endDate} is:issue` }, - { type: 'pull-request', query: `involves:${username} updated:${startDate}..${endDate} is:pull-request` }, + { type: 'issue', query: buildCombinedQuery('issue') }, + { type: 'pull-request', query: buildCombinedQuery('pull-request') }, ]; for (const { type, query: searchQuery } of queries) { @@ -153,7 +164,8 @@ export const useGitHubDataFetching = ({ while (page <= GITHUB_SEARCH_API_MAX_PAGES) { try { const typeLabel = type === 'pull-request' ? 'PRs' : 'issues'; - onProgress(`Fetching ${typeLabel} page ${page} for ${username}...`); + const usersLabel = usernames.length === 1 ? usernames[0] : `${usernames.length} users`; + onProgress(`Fetching ${typeLabel} page ${page} for ${usersLabel}...`); const response = await fetch( `https://api.github.com/search/issues?q=${encodeURIComponent(searchQuery)}&per_page=${perPage}&page=${page}&sort=updated`, @@ -171,7 +183,7 @@ export const useGitHubDataFetching = ({ // Handle pagination limit error (422) - continue with next query if (response.status === 422 && responseJSON.message?.includes('pagination')) { console.warn( - `GitHub Search API pagination limit reached for ${username} ${type} at page ${page}.` + `GitHub Search API pagination limit reached for ${type} at page ${page}.` ); break; } @@ -200,7 +212,6 @@ export const useGitHubDataFetching = ({ } // Check if we've fetched all items for this query - // Stop if we got fewer items than requested OR if we've fetched all available items for this query if (items.length < perPage || itemsFetchedForQuery >= totalCount) { break; } @@ -211,7 +222,7 @@ export const useGitHubDataFetching = ({ await new Promise(resolve => setTimeout(resolve, GITHUB_API_DELAY_MS)); } catch (error) { - console.error(`Error fetching ${type} page ${page} for ${username}:`, error); + console.error(`Error fetching ${type} page ${page}:`, error); // Return partial results if we have any, otherwise throw if (allItems.length > 0) { console.warn(`Returning ${allItems.length} items collected before error`); @@ -299,28 +310,30 @@ export const useGitHubDataFetching = ({ // Accumulate all data from all users const allEvents: GitHubEvent[] = []; - const allSearchItems: GitHubItem[] = []; - // Fetch events for each username with pagination + // Fetch all issues/PRs for all users in combined queries (2 API calls total) + setCurrentUsername(usernames.length === 1 ? usernames[0] : `${usernames.length} users`); + let allSearchItems: GitHubItem[] = []; + try { + allSearchItems = await fetchAllSearchItemsForUsers(usernames, githubToken, startDate, endDate, onProgress); + onProgress(`Fetched ${allSearchItems.length} issues/PRs for ${usernames.length} user(s)`); + } catch (error) { + console.error('Error fetching issues/PRs:', error); + onError(`Error fetching issues/PRs: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + + // Fetch events for each username (Events API doesn't support OR queries) for (const singleUsername of usernames) { setCurrentUsername(singleUsername); try { - // Fetch all issues and PRs with pagination - const userSearchItems = await fetchAllSearchItems(singleUsername, githubToken, startDate, endDate, onProgress); - allSearchItems.push(...userSearchItems); - onProgress(`Fetched ${userSearchItems.length} issues/PRs for ${singleUsername}`); - - // Fetch all events with pagination - const userEvents = await fetchAllEvents(singleUsername, githubToken, startDate, endDate, onProgress); - allEvents.push(...userEvents); - onProgress(`Fetched ${userEvents.length} events for ${singleUsername}`); - - + const userEvents = await fetchAllEvents(singleUsername, githubToken, startDate, endDate, onProgress); + allEvents.push(...userEvents); + onProgress(`Fetched ${userEvents.length} events for ${singleUsername}`); } catch (error) { - console.error(`Error fetching data for ${singleUsername}:`, error); + console.error(`Error fetching events for ${singleUsername}:`, error); onError( - `Error fetching data for ${singleUsername}: ${error instanceof Error ? error.message : 'Unknown error'}` + `Error fetching events for ${singleUsername}: ${error instanceof Error ? error.message : 'Unknown error'}` ); // Continue with other usernames instead of breaking continue; From edf6479102ad1faae7e80946defcdd12c836c9ab Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 23:16:57 +0000 Subject: [PATCH 13/14] Revert to per-user queries with involves: qualifier Combined OR queries were rejected by GitHub API with "search contains only logical operators without search terms" error. Reverted to simple per-user queries using involves: qualifier (2 API calls per user). The involves: qualifier still provides optimization by matching author, assignee, commenter, and mentions in a single query instead of separate author/assignee queries. --- src/hooks/useGitHubDataFetching.ts | 61 ++++++++++-------------------- 1 file changed, 19 insertions(+), 42 deletions(-) diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index 4c8c801..b39b1ac 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -127,10 +127,9 @@ export const useGitHubDataFetching = ({ return allEvents; }; - // Fetch all search items (issues/PRs) for multiple usernames with pagination - // Combines users into single queries to reduce API calls - const fetchAllSearchItemsForUsers = async ( - usernames: string[], + // Fetch all search items (issues/PRs) for a username with pagination + const fetchAllSearchItems = async ( + username: string, token: string, startDate: string, endDate: string, @@ -140,21 +139,10 @@ export const useGitHubDataFetching = ({ const seenIds = new Set(); const perPage = GITHUB_API_PER_PAGE; - // Build combined query for all users using OR - // Format: "is:issue updated:... involves:user1 OR is:issue updated:... involves:user2" - // This works because AND has higher precedence than OR in GitHub search - const buildCombinedQuery = (type: string) => { - const typeFilter = type === 'issue' ? 'is:issue' : 'is:pull-request'; - const dateFilter = `updated:${startDate}..${endDate}`; - - return usernames - .map(user => `${typeFilter} ${dateFilter} involves:${user}`) - .join(' OR '); - }; - + // Use "involves:" to match author, assignee, commenter, and mentions in one query const queries = [ - { type: 'issue', query: buildCombinedQuery('issue') }, - { type: 'pull-request', query: buildCombinedQuery('pull-request') }, + { type: 'issue', query: `is:issue updated:${startDate}..${endDate} involves:${username}` }, + { type: 'pull-request', query: `is:pull-request updated:${startDate}..${endDate} involves:${username}` }, ]; for (const { type, query: searchQuery } of queries) { @@ -164,8 +152,7 @@ export const useGitHubDataFetching = ({ while (page <= GITHUB_SEARCH_API_MAX_PAGES) { try { const typeLabel = type === 'pull-request' ? 'PRs' : 'issues'; - const usersLabel = usernames.length === 1 ? usernames[0] : `${usernames.length} users`; - onProgress(`Fetching ${typeLabel} page ${page} for ${usersLabel}...`); + onProgress(`Fetching ${typeLabel} page ${page} for ${username}...`); const response = await fetch( `https://api.github.com/search/issues?q=${encodeURIComponent(searchQuery)}&per_page=${perPage}&page=${page}&sort=updated`, @@ -182,9 +169,7 @@ export const useGitHubDataFetching = ({ // Handle pagination limit error (422) - continue with next query if (response.status === 422 && responseJSON.message?.includes('pagination')) { - console.warn( - `GitHub Search API pagination limit reached for ${type} at page ${page}.` - ); + console.warn(`GitHub Search API pagination limit reached for ${username} ${type} at page ${page}.`); break; } @@ -195,7 +180,6 @@ export const useGitHubDataFetching = ({ const totalCount = searchData.total_count; const items = searchData.items; - // Track items fetched for this query (before deduplication) itemsFetchedForQuery += items.length; // Add items, deduplicating by id @@ -211,18 +195,15 @@ export const useGitHubDataFetching = ({ } } - // Check if we've fetched all items for this query if (items.length < perPage || itemsFetchedForQuery >= totalCount) { break; } page++; - - // Add a small delay to respect GitHub API rate limits await new Promise(resolve => setTimeout(resolve, GITHUB_API_DELAY_MS)); } catch (error) { - console.error(`Error fetching ${type} page ${page}:`, error); + console.error(`Error fetching ${type} page ${page} for ${username}:`, error); // Return partial results if we have any, otherwise throw if (allItems.length > 0) { console.warn(`Returning ${allItems.length} items collected before error`); @@ -310,30 +291,26 @@ export const useGitHubDataFetching = ({ // Accumulate all data from all users const allEvents: GitHubEvent[] = []; + const allSearchItems: GitHubItem[] = []; - // Fetch all issues/PRs for all users in combined queries (2 API calls total) - setCurrentUsername(usernames.length === 1 ? usernames[0] : `${usernames.length} users`); - let allSearchItems: GitHubItem[] = []; - try { - allSearchItems = await fetchAllSearchItemsForUsers(usernames, githubToken, startDate, endDate, onProgress); - onProgress(`Fetched ${allSearchItems.length} issues/PRs for ${usernames.length} user(s)`); - } catch (error) { - console.error('Error fetching issues/PRs:', error); - onError(`Error fetching issues/PRs: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - - // Fetch events for each username (Events API doesn't support OR queries) + // Fetch data for each username for (const singleUsername of usernames) { setCurrentUsername(singleUsername); try { + // Fetch issues/PRs (2 API calls per user using involves:) + const userSearchItems = await fetchAllSearchItems(singleUsername, githubToken, startDate, endDate, onProgress); + allSearchItems.push(...userSearchItems); + onProgress(`Fetched ${userSearchItems.length} issues/PRs for ${singleUsername}`); + + // Fetch events const userEvents = await fetchAllEvents(singleUsername, githubToken, startDate, endDate, onProgress); allEvents.push(...userEvents); onProgress(`Fetched ${userEvents.length} events for ${singleUsername}`); } catch (error) { - console.error(`Error fetching events for ${singleUsername}:`, error); + console.error(`Error fetching data for ${singleUsername}:`, error); onError( - `Error fetching events for ${singleUsername}: ${error instanceof Error ? error.message : 'Unknown error'}` + `Error fetching data for ${singleUsername}: ${error instanceof Error ? error.message : 'Unknown error'}` ); // Continue with other usernames instead of breaking continue; From bf803ceca2deb073f22c7500497343369bfc0fb7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 23:24:01 +0000 Subject: [PATCH 14/14] Optimize queries using involves: qualifier Combines multiple users into single queries using OR between involves: Query: "is:issue updated:... involves:user1 OR involves:user2" This reduces N users from 2*N API calls to just 2 calls total for issues/PRs. Events API still requires per-user calls. --- src/hooks/useGitHubDataFetching.ts | 47 ++++++++++++++++++------------ 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/hooks/useGitHubDataFetching.ts b/src/hooks/useGitHubDataFetching.ts index b39b1ac..f2d48ac 100644 --- a/src/hooks/useGitHubDataFetching.ts +++ b/src/hooks/useGitHubDataFetching.ts @@ -127,9 +127,10 @@ export const useGitHubDataFetching = ({ return allEvents; }; - // Fetch all search items (issues/PRs) for a username with pagination + // Fetch all search items (issues/PRs) for multiple usernames with pagination + // Combines users with OR in a single query to reduce API calls const fetchAllSearchItems = async ( - username: string, + usernames: string[], token: string, startDate: string, endDate: string, @@ -139,20 +140,24 @@ export const useGitHubDataFetching = ({ const seenIds = new Set(); const perPage = GITHUB_API_PER_PAGE; - // Use "involves:" to match author, assignee, commenter, and mentions in one query + // Build query with multiple involves: joined by OR + // Format: "is:issue updated:... involves:user1 OR involves:user2 OR involves:user3" + const involvesClause = usernames.map(u => `involves:${u}`).join(' OR '); + const queries = [ - { type: 'issue', query: `is:issue updated:${startDate}..${endDate} involves:${username}` }, - { type: 'pull-request', query: `is:pull-request updated:${startDate}..${endDate} involves:${username}` }, + { type: 'issue', query: `is:issue updated:${startDate}..${endDate} ${involvesClause}` }, + { type: 'pull-request', query: `is:pull-request updated:${startDate}..${endDate} ${involvesClause}` }, ]; for (const { type, query: searchQuery } of queries) { let page = 1; let itemsFetchedForQuery = 0; + const usersLabel = usernames.length === 1 ? usernames[0] : `${usernames.length} users`; while (page <= GITHUB_SEARCH_API_MAX_PAGES) { try { const typeLabel = type === 'pull-request' ? 'PRs' : 'issues'; - onProgress(`Fetching ${typeLabel} page ${page} for ${username}...`); + onProgress(`Fetching ${typeLabel} page ${page} for ${usersLabel}...`); const response = await fetch( `https://api.github.com/search/issues?q=${encodeURIComponent(searchQuery)}&per_page=${perPage}&page=${page}&sort=updated`, @@ -169,7 +174,7 @@ export const useGitHubDataFetching = ({ // Handle pagination limit error (422) - continue with next query if (response.status === 422 && responseJSON.message?.includes('pagination')) { - console.warn(`GitHub Search API pagination limit reached for ${username} ${type} at page ${page}.`); + console.warn(`GitHub Search API pagination limit reached for ${type} at page ${page}.`); break; } @@ -203,7 +208,7 @@ export const useGitHubDataFetching = ({ await new Promise(resolve => setTimeout(resolve, GITHUB_API_DELAY_MS)); } catch (error) { - console.error(`Error fetching ${type} page ${page} for ${username}:`, error); + console.error(`Error fetching ${type} page ${page}:`, error); // Return partial results if we have any, otherwise throw if (allItems.length > 0) { console.warn(`Returning ${allItems.length} items collected before error`); @@ -291,28 +296,32 @@ export const useGitHubDataFetching = ({ // Accumulate all data from all users const allEvents: GitHubEvent[] = []; - const allSearchItems: GitHubItem[] = []; - // Fetch data for each username + // Fetch all issues/PRs for all users in a single combined query (2 API calls total) + // Query format: "is:issue updated:... involves:user1 OR involves:user2 OR involves:user3" + setCurrentUsername(usernames.length === 1 ? usernames[0] : `${usernames.length} users`); + let allSearchItems: GitHubItem[] = []; + try { + allSearchItems = await fetchAllSearchItems(usernames, githubToken, startDate, endDate, onProgress); + onProgress(`Fetched ${allSearchItems.length} issues/PRs for ${usernames.length} user(s)`); + } catch (error) { + console.error('Error fetching issues/PRs:', error); + onError(`Error fetching issues/PRs: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + + // Fetch events for each username (Events API doesn't support combined queries) for (const singleUsername of usernames) { setCurrentUsername(singleUsername); try { - // Fetch issues/PRs (2 API calls per user using involves:) - const userSearchItems = await fetchAllSearchItems(singleUsername, githubToken, startDate, endDate, onProgress); - allSearchItems.push(...userSearchItems); - onProgress(`Fetched ${userSearchItems.length} issues/PRs for ${singleUsername}`); - - // Fetch events const userEvents = await fetchAllEvents(singleUsername, githubToken, startDate, endDate, onProgress); allEvents.push(...userEvents); onProgress(`Fetched ${userEvents.length} events for ${singleUsername}`); } catch (error) { - console.error(`Error fetching data for ${singleUsername}:`, error); + console.error(`Error fetching events for ${singleUsername}:`, error); onError( - `Error fetching data for ${singleUsername}: ${error instanceof Error ? error.message : 'Unknown error'}` + `Error fetching events for ${singleUsername}: ${error instanceof Error ? error.message : 'Unknown error'}` ); - // Continue with other usernames instead of breaking continue; } }