From aee09192507dcdac864cf2b995c706881dd2a09d Mon Sep 17 00:00:00 2001 From: filip mertens Date: Tue, 23 Apr 2024 11:50:33 +0200 Subject: [PATCH] paged fetching data --- apps/remix-ide/src/app/files/dgitProvider.ts | 32 +++-- .../src/components/github/branchselect.tsx | 49 +++++++ .../components/github/repositoryselect.tsx | 44 +++--- .../github/selectandclonerepositories.tsx | 55 ++++++++ .../git/src/components/panels/clone.tsx | 4 +- .../src/components/panels/commands/merge.tsx | 1 - .../src/components/panels/remotesimport.tsx | 35 ++--- .../src/components/panels/repositories.tsx | 125 ------------------ libs/remix-ui/git/src/lib/gitactions.ts | 29 +++- 9 files changed, 185 insertions(+), 189 deletions(-) create mode 100644 libs/remix-ui/git/src/components/github/branchselect.tsx create mode 100644 libs/remix-ui/git/src/components/github/selectandclonerepositories.tsx delete mode 100644 libs/remix-ui/git/src/components/panels/repositories.tsx diff --git a/apps/remix-ide/src/app/files/dgitProvider.ts b/apps/remix-ide/src/app/files/dgitProvider.ts index 65728a3382..7473afbcb6 100644 --- a/apps/remix-ide/src/app/files/dgitProvider.ts +++ b/apps/remix-ide/src/app/files/dgitProvider.ts @@ -984,7 +984,7 @@ class DGitProvider extends Plugin { // OCTOKIT FEATURES - async remotebranches(input: { owner: string, repo: string, token: string }) { + async remotebranches(input: { owner: string, repo: string, token: string, page: number, per_page: number }) { console.log(input) const octokit = new Octokit({ @@ -994,9 +994,10 @@ class DGitProvider extends Plugin { const data = await octokit.request('GET /repos/{owner}/{repo}/branches{?protected,per_page,page}', { owner: input.owner, repo: input.repo, - per_page: 100 + per_page: input.per_page || 100, + page: input.page || 1 }) - console.log(data) + return data.data } @@ -1090,20 +1091,31 @@ class DGitProvider extends Plugin { return pages } - async repositories(input: { token: string }) { - // Set your access token + async repositories(input: { token: string, page?: number, per_page?: number }) { + const accessToken = input.token; - // Set headers for the request + let page = input.page || 1 + const perPage = input.per_page || 10 + + const baseURL = 'https://api.github.com/user/repos' + const repositories = [] + const sort = 'updated' + const direction = 'desc' + const headers = { 'Authorization': `Bearer ${accessToken}`, // Include your GitHub access token 'Accept': 'application/vnd.github.v3+json', // GitHub API v3 media type }; - // Make the GET request with Axios - const response = await axios.get('https://api.github.com/user/repos?visibility=private,public', { headers }) - - return response.data + + const url = `${baseURL}?visibility=private,public&page=${page}&per_page=${perPage}&sort=${sort}&direction=${direction}`; + const response = await axios.get(url, { headers }); + + repositories.push(...response.data); + + + return repositories } } diff --git a/libs/remix-ui/git/src/components/github/branchselect.tsx b/libs/remix-ui/git/src/components/github/branchselect.tsx new file mode 100644 index 0000000000..4dcda8c2d8 --- /dev/null +++ b/libs/remix-ui/git/src/components/github/branchselect.tsx @@ -0,0 +1,49 @@ +import React, { useState, useCallback, useEffect } from 'react'; +import Select from 'react-select'; +import { gitActionsContext } from '../../state/context'; +import { selectStyles, selectTheme } from '../../types/styles'; +import { gitPluginContext } from '../gitui'; + +interface BranchySelectProps { + select: (branch:{ name: string }) => void; +} + +export const BranchSelect = (props: BranchySelectProps) => { + const context = React.useContext(gitPluginContext) + const actions = React.useContext(gitActionsContext) + const [branchOptions, setBranchOptions] = useState([]); + + useEffect(() => { + if (context.remoteBranches && context.remoteBranches.length > 0) { + const options = context.remoteBranches && context.remoteBranches.length > 0 && context.remoteBranches.map(branch => { + return { value: branch.name, label: branch.name } + } + ) + setBranchOptions(options) + } else { + setBranchOptions(null) + } + }, [context.remoteBranches]) + + const selectRemoteBranch = async (e: any) => { + if(!e || !e.value) { + props.select(null) + return + } + const value = e && e.value + props.select({ name: value.toString() }) + } + + return (<>{branchOptions && branchOptions.length ? + e && selectRepo(e.value)} + onChange={(e: any) => selectRepo(e)} theme={selectTheme} styles={selectStyles} isClearable={true} diff --git a/libs/remix-ui/git/src/components/github/selectandclonerepositories.tsx b/libs/remix-ui/git/src/components/github/selectandclonerepositories.tsx new file mode 100644 index 0000000000..9437d9277f --- /dev/null +++ b/libs/remix-ui/git/src/components/github/selectandclonerepositories.tsx @@ -0,0 +1,55 @@ +import React, { useEffect, useState } from "react"; +import { gitActionsContext } from "../../state/context"; +import { repository } from "../../types"; +import { gitPluginContext } from "../gitui"; +import RepositorySelect from "./repositoryselect"; +import { BranchSelect } from "./branchselect"; +import { TokenWarning } from "../panels/tokenWarning"; + +interface RepositoriesProps { + cloneDepth?: number + cloneAllBranches?: boolean +} + +export const SelectAndCloneRepositories = (props: RepositoriesProps) => { + const { cloneDepth, cloneAllBranches } = props + const context = React.useContext(gitPluginContext) + const actions = React.useContext(gitActionsContext) + const [branch, setBranch] = useState({ name: "" }); + const [repo, setRepo] = useState(null); + + const selectRemoteBranch = async (branch:{ name: string }) => { + setBranch(branch) + } + + const selectRepo = async (repo: repository) => { + setBranch(null) + setRepo(repo) + } + + const clone = async () => { + try { + actions.clone(repo.html_url, branch.name, cloneDepth, !cloneAllBranches) + } catch (e) { + // do nothing + } + }; + + + return ( + <> + + + + {repo &&} + + {repo && branch && branch.name && branch.name !== '0' ? + : null} + + + ) +} + + diff --git a/libs/remix-ui/git/src/components/panels/clone.tsx b/libs/remix-ui/git/src/components/panels/clone.tsx index 1d1bab91cc..dd501df2ff 100644 --- a/libs/remix-ui/git/src/components/panels/clone.tsx +++ b/libs/remix-ui/git/src/components/panels/clone.tsx @@ -4,7 +4,7 @@ import { Alert, Form, FormControl, InputGroup } from "react-bootstrap"; import { useLocalStorage } from "../../hooks/useLocalStorage"; import { gitActionsContext } from "../../state/context"; import { gitPluginContext } from "../gitui"; -import { Repositories } from "./repositories"; +import { SelectAndCloneRepositories } from "../github/selectandclonerepositories"; import { RemixUiCheckbox } from "@remix-ui/checkbox"; import GitUIButton from "../buttons/gituibutton"; @@ -69,7 +69,7 @@ export const Clone = () => { clone() }}>clone
- +
diff --git a/libs/remix-ui/git/src/components/panels/commands/merge.tsx b/libs/remix-ui/git/src/components/panels/commands/merge.tsx index 431a388010..e50705553c 100644 --- a/libs/remix-ui/git/src/components/panels/commands/merge.tsx +++ b/libs/remix-ui/git/src/components/panels/commands/merge.tsx @@ -27,7 +27,6 @@ export const Merge = () => { } useEffect(() => { - console.log('context', context.repositories) // map context.repositories to options const localBranches = context.branches && context.branches.length > 0 && context.branches .filter(branch => !branch.remote) diff --git a/libs/remix-ui/git/src/components/panels/remotesimport.tsx b/libs/remix-ui/git/src/components/panels/remotesimport.tsx index 41c9de6b8c..3706b0a320 100644 --- a/libs/remix-ui/git/src/components/panels/remotesimport.tsx +++ b/libs/remix-ui/git/src/components/panels/remotesimport.tsx @@ -6,6 +6,7 @@ import { gitPluginContext } from "../gitui"; import Select from 'react-select' import { selectStyles, selectTheme } from "../../types/styles"; import { TokenWarning } from "./tokenWarning"; +import RepositorySelect from "../github/repositoryselect"; export const RemotesImport = () => { @@ -18,7 +19,6 @@ export const RemotesImport = () => { const [remoteName, setRemoteName] = useState('') useEffect(() => { - console.log('context', context.repositories) if (context.repositories && context.repositories.length > 0) { // map context.repositories to options const options = context.repositories && context.repositories.length > 0 && context.repositories.map(repo => { @@ -44,19 +44,15 @@ export const RemotesImport = () => { } }; - const selectRepo = async (value: number | string) => { - // find repo - console.log('setRepo', value, context.repositories) - - const repo = context.repositories.find(repo => { - return repo.id.toString() === value.toString() - }) - console.log('repo', repo) - if (repo) { - setRepo(repo) - } + const selectRepo = async (repo: repository) => { + setRepo(repo) } + + useEffect(() => { + console.log('OTIONS', repoOtions) + },[repoOtions]) + const addRemote = async () => { try { actions.addRemote({ @@ -75,20 +71,7 @@ export const RemotesImport = () => { return ( <> - - {show ? - onRemoteNameChange(e.target.value)} value={remoteName} className="form-control mb-2" type="text" id="remotename" /> diff --git a/libs/remix-ui/git/src/components/panels/repositories.tsx b/libs/remix-ui/git/src/components/panels/repositories.tsx deleted file mode 100644 index 6ab5f32d37..0000000000 --- a/libs/remix-ui/git/src/components/panels/repositories.tsx +++ /dev/null @@ -1,125 +0,0 @@ -import React, { useEffect, useState } from "react"; -import { Alert, Button } from "react-bootstrap"; -import { gitActionsContext } from "../../state/context"; -import { repository } from "../../types"; -import { gitPluginContext } from "../gitui"; -import AsyncSelect from 'react-select' -import { selectStyles, selectTheme } from "../../types/styles"; -import { TokenWarning } from "./tokenWarning"; -import RepositorySelect from "../github/repositoryselect"; - -interface RepositoriesProps { - cloneDepth?: number - cloneAllBranches?: boolean -} - -export const Repositories = (props: RepositoriesProps) => { - const { cloneDepth, cloneAllBranches } = props - const context = React.useContext(gitPluginContext) - const actions = React.useContext(gitActionsContext) - const [branch, setBranch] = useState({ name: "" }); - const [repo, setRepo] = useState(null); - const [repoOtions, setRepoOptions] = useState([]); - const [branchOptions, setBranchOptions] = useState([]); - const [loading, setLoading] = useState(false) - const [show, setShow] = useState(false) - - useEffect(() => { - console.log('context', context.repositories) - if (context.repositories && context.repositories.length > 0) { - // map context.repositories to options - const options = context.repositories && context.repositories.length > 0 && context.repositories.map(repo => { - return { value: repo.id, label: repo.full_name } - }) - setRepoOptions(options) - } else { - setRepoOptions(null) - setShow(false) - } - setLoading(false) - - }, [context.repositories]) - - - useEffect(() => { - // map context.branches to options - const options = context.remoteBranches && context.remoteBranches.length > 0 && context.remoteBranches.map(branch => { - return { value: branch.name, label: branch.name } - } - ) - setBranchOptions(options) - }, [context.remoteBranches]) - - - useEffect(() => { - console.log('show', show) - },[show]) - - const fetchRepositories = async () => { - try { - setShow(true) - setLoading(true) - setRepoOptions([]) - setBranchOptions([]) - console.log(await actions.repositories()) - } catch (e) { - // do nothing - } - }; - - const selectRepo = async (value: number | string) => { - // find repo - console.log('setRepo', value, context.repositories) - - const repo = context.repositories.find(repo => { - return repo.id.toString() === value.toString() - }) - console.log('repo', repo) - if (repo) { - setBranchOptions([]) - setBranch({ name: "" }) - setRepo(repo) - await actions.remoteBranches(repo.owner.login, repo.name) - } - } - - const selectRemoteBranch = async (value: number | string) => { - console.log('setRemoteBranch', value) - setBranch({ name: value.toString() }) - } - - const clone = async () => { - try { - console.log('clone', repo, branch) - actions.clone(repo.html_url, branch.name, cloneDepth, !cloneAllBranches) - } catch (e) { - // do nothing - } - }; - - - return ( - <> - - - {branchOptions && branchOptions.length ? - e && selectRemoteBranch(e.value)} - theme={selectTheme} - styles={selectStyles} - isClearable={true} - placeholder="Type to search for a branch..." - /> : null} - - {repo && branch.name && branch.name !== '0' ? - : null} - - - ) -} - - diff --git a/libs/remix-ui/git/src/lib/gitactions.ts b/libs/remix-ui/git/src/lib/gitactions.ts index f19c0075bf..1a15191fa5 100644 --- a/libs/remix-ui/git/src/lib/gitactions.ts +++ b/libs/remix-ui/git/src/lib/gitactions.ts @@ -470,8 +470,21 @@ export const repositories = async () => { try { const token = await tokenWarning(); if (token) { - const repos = await plugin.call('dGitProvider' as any, 'repositories', { token }); + let repos = await plugin.call('dGitProvider' as any, 'repositories', { token, per_page: 100 }) dispatch(setRepos(repos)) + let page = 2 + let hasMoreData = true + const per_page = 100 + while(hasMoreData){ + let pagedResponse = await plugin.call('dGitProvider' as any, 'repositories', { token, page:page, per_page: per_page }) + if(pagedResponse.length < per_page){ + hasMoreData = false + } + repos = [...repos,...pagedResponse] + dispatch(setRepos(repos)) + page++ + } + } else { plugin.call('notification', 'alert', { title: 'Error getting repositories', @@ -493,8 +506,20 @@ export const remoteBranches = async (owner: string, repo: string) => { try { const token = await tokenWarning(); if (token) { - const branches = await plugin.call('dGitProvider' as any, 'remotebranches', { token, owner, repo }); + let branches = await plugin.call('dGitProvider' as any, 'remotebranches', { token, owner, repo, per_page: 100 }); dispatch(setRemoteBranches(branches)) + let page = 2 + let hasMoreData = true + const per_page = 100 + while(hasMoreData){ + let pagedResponse = await plugin.call('dGitProvider' as any, 'remotebranches', { token, owner, repo, page:page, per_page: per_page }) + if(pagedResponse.length < per_page){ + hasMoreData = false + } + branches = [...branches,...pagedResponse] + dispatch(setRemoteBranches(branches)) + page++ + } } else { plugin.call('notification', 'alert', { title: 'Error getting branches',