paged fetching data

apps/remix-ide/src/app/files/dgitProvider.ts

@@ -984,7 +984,7 @@ class DGitProvider extends Plugin {
 
   // OCTOKIT FEATURES
 
-  async remotebranches(input: { owner: string, repo: string, token: string }) {
+  async remotebranches(input: { owner: string, repo: string, token: string, page: number, per_page:  number }) {
     console.log(input)
 
     const octokit = new Octokit({
@@ -994,9 +994,10 @@ class DGitProvider extends Plugin {
     const data = await octokit.request('GET /repos/{owner}/{repo}/branches{?protected,per_page,page}', {
       owner: input.owner,
       repo: input.repo,
-      per_page: 100
+      per_page: input.per_page || 100,
+      page: input.page || 1
     })
-    console.log(data)
+
     return data.data
   }
 
@@ -1090,20 +1091,31 @@ class DGitProvider extends Plugin {
     return pages
   }
 
-  async repositories(input: { token: string }) {
+  async repositories(input: { token: string, page?: number, per_page?: number }) {
-    // Set your access token
+
     const accessToken = input.token;
 
-    // Set headers for the request
+    let page = input.page || 1
+    const perPage = input.per_page || 10
+
+    const baseURL = 'https://api.github.com/user/repos'
+    const repositories = []
+    const sort = 'updated'
+    const direction = 'desc'
+
     const headers = {
       'Authorization': `Bearer ${accessToken}`, // Include your GitHub access token
       'Accept': 'application/vnd.github.v3+json', // GitHub API v3 media type
     };
 
-    // Make the GET request with Axios
-    const response = await axios.get('https://api.github.com/user/repos?visibility=private,public', { headers })
 
-    return response.data
+    const url = `${baseURL}?visibility=private,public&page=${page}&per_page=${perPage}&sort=${sort}&direction=${direction}`;
+    const response = await axios.get(url, { headers });
+
+    repositories.push(...response.data);
+
+
+    return repositories
   }
 
 }

libs/remix-ui/git/src/components/github/branchselect.tsx

@@ -0,0 +1,49 @@
+import React, { useState, useCallback, useEffect } from 'react';
+import Select from 'react-select';
+import { gitActionsContext } from '../../state/context';
+import { selectStyles, selectTheme } from '../../types/styles';
+import { gitPluginContext } from '../gitui';
+
+interface BranchySelectProps {
+  select: (branch:{ name: string }) => void;
+}
+
+export const BranchSelect = (props: BranchySelectProps) => {
+  const context = React.useContext(gitPluginContext)
+  const actions = React.useContext(gitActionsContext)
+  const [branchOptions, setBranchOptions] = useState<any>([]);
+
+  useEffect(() => {
+    if (context.remoteBranches && context.remoteBranches.length > 0) {
+      const options = context.remoteBranches && context.remoteBranches.length > 0 && context.remoteBranches.map(branch => {
+        return { value: branch.name, label: branch.name }
+      }
+      )
+      setBranchOptions(options)
+    } else {
+      setBranchOptions(null)
+    }
+  }, [context.remoteBranches])
+
+  const selectRemoteBranch = async (e: any) => {
+    if(!e || !e.value) {
+      props.select(null) 
+      return
+    }
+    const value = e && e.value
+    props.select({ name: value.toString() })
+  }
+
+  return (<>{branchOptions && branchOptions.length ?
+    <Select
+      options={branchOptions}
+      className="mt-1"
+      onChange={(e: any) =>selectRemoteBranch(e)}
+      theme={selectTheme}
+      styles={selectStyles}
+      isClearable={true}
+      placeholder="Type to search for a branch..."
+    /> : null}
+  </>)
+
+}

libs/remix-ui/git/src/components/github/repositoryselect.tsx

@@ -1,15 +1,17 @@
-import { clearInstances } from 'libs/remix-ui/run-tab/src/lib/actions/actions';
+import React, { useState, useEffect } from 'react';
-import React, { useState, useCallback, useEffect } from 'react';
 import { Button } from 'react-bootstrap';
-import { OptionsOrGroups, GroupBase } from 'react-select';
+import Select from 'react-select';
-import Select from 'react-select/async';
 import { gitActionsContext } from '../../state/context';
+import { repository } from '../../types';
 import { selectStyles, selectTheme } from '../../types/styles';
 import { gitPluginContext } from '../gitui';
 import { TokenWarning } from '../panels/tokenWarning';
 
-const RepositorySelect = () => {
+interface RepositorySelectProps {
-  const [page, setPage] = useState(1);
+  select: (repo: repository) => void;
+}
+
+const RepositorySelect = (props: RepositorySelectProps) => {
   const [repoOtions, setRepoOptions] = useState<any>([]);
   const context = React.useContext(gitPluginContext)
   const actions = React.useContext(gitActionsContext)
@@ -17,14 +19,14 @@ const RepositorySelect = () => {
   const [show, setShow] = useState(false)
 
   useEffect(() => {
-    console.log('context', context.repositories)
     if (context.repositories && context.repositories.length > 0) {
       // map context.repositories to options
       const options = context.repositories && context.repositories.length > 0 && context.repositories.map(repo => {
         return { value: repo.id, label: repo.full_name }
       })
-      console.log('options', options)
+
       setRepoOptions(options)
+      setShow(options.length > 0)
     } else {
       setRepoOptions(null)
       setShow(false)
@@ -33,22 +35,19 @@ const RepositorySelect = () => {
 
   }, [context.repositories])
 
-  useEffect(() => {
+  const selectRepo = async (e: any) => {
-    console.log('OTIONS', repoOtions)
+    if(!e || !e.value) {
-  },[repoOtions])
+      props.select(null) 
-
+      return
-  const selectRepo = async (value: number | string) => {
+    }
-    // find repo
+    const value = e && e.value
-    console.log('setRepo', value, context.repositories)
   
     const repo = context.repositories.find(repo => {
       return repo.id.toString() === value.toString()
     })
-    console.log('repo', repo)
+    
     if (repo) {
-      //setBranchOptions([])
+      props.select(repo)
-      //setBranch({ name: "" })
-      //setRepo(repo)
       await actions.remoteBranches(repo.owner.login, repo.name)
     }
   }
@@ -57,8 +56,7 @@ const RepositorySelect = () => {
     try {
       setShow(true)
       setLoading(true)
-      //setRepoOptions([])
+      setRepoOptions([])
-      //setBranchOptions([])
       console.log(await actions.repositories())
     } catch (e) {
       // do nothing
@@ -74,7 +72,7 @@ const RepositorySelect = () => {
           <Select
             options={repoOtions}
             className="mt-1"
-            onChange={(e: any) => e && selectRepo(e.value)}
+            onChange={(e: any) => selectRepo(e)}
             theme={selectTheme}
             styles={selectStyles}
             isClearable={true}

libs/remix-ui/git/src/components/github/selectandclonerepositories.tsx

@@ -0,0 +1,55 @@
+import React, { useEffect, useState } from "react";
+import { gitActionsContext } from "../../state/context";
+import { repository } from "../../types";
+import { gitPluginContext } from "../gitui";
+import RepositorySelect from "./repositoryselect";
+import { BranchSelect } from "./branchselect";
+import { TokenWarning } from "../panels/tokenWarning";
+
+interface RepositoriesProps {
+  cloneDepth?: number
+  cloneAllBranches?: boolean
+}
+
+export const SelectAndCloneRepositories = (props: RepositoriesProps) => {
+  const { cloneDepth, cloneAllBranches } = props
+  const context = React.useContext(gitPluginContext)
+  const actions = React.useContext(gitActionsContext)
+  const [branch, setBranch] = useState({ name: "" });
+  const [repo, setRepo] = useState<repository>(null);
+
+  const selectRemoteBranch = async (branch:{ name: string }) => {
+    setBranch(branch)
+  }
+
+  const selectRepo = async (repo: repository) => {
+    setBranch(null)
+    setRepo(repo)
+  }
+
+  const clone = async () => {
+    try {
+      actions.clone(repo.html_url, branch.name, cloneDepth, !cloneAllBranches)
+    } catch (e) {
+      // do nothing
+    }
+  };
+
+  
+  return (
+    <>
+      <TokenWarning />
+      <RepositorySelect select={selectRepo} />
+
+      {repo &&<BranchSelect select={selectRemoteBranch} />}
+
+      {repo && branch && branch.name && branch.name !== '0' ?
+        <button data-id='clonebtn' className='btn btn-primary mt-1 w-100' onClick={async () => {
+          await clone()
+        }}>clone {repo.full_name}:{branch.name}</button> : null}
+
+    </>
+  )
+}
+
+

libs/remix-ui/git/src/components/panels/clone.tsx

@@ -4,7 +4,7 @@ import { Alert, Form, FormControl, InputGroup } from "react-bootstrap";
 import { useLocalStorage } from "../../hooks/useLocalStorage";
 import { gitActionsContext } from "../../state/context";
 import { gitPluginContext } from "../gitui";
-import { Repositories } from "./repositories";
+import { SelectAndCloneRepositories } from "../github/selectandclonerepositories";
 import { RemixUiCheckbox } from "@remix-ui/checkbox";
 import GitUIButton from "../buttons/gituibutton";
 
@@ -69,7 +69,7 @@ export const Clone = () => {
                 clone()
             }}>clone</GitUIButton>
             <hr />
-            <Repositories cloneAllBranches={cloneAllBranches} cloneDepth={cloneDepth} />
+            <SelectAndCloneRepositories cloneAllBranches={cloneAllBranches} cloneDepth={cloneDepth} />
             <hr />
             <label>options</label>
             <InputGroup className="mt-1 mb-1">

libs/remix-ui/git/src/components/panels/commands/merge.tsx

@@ -27,7 +27,6 @@ export const Merge = () => {
   }
 
   useEffect(() => {
-    console.log('context', context.repositories)
     // map context.repositories to options
     const localBranches = context.branches && context.branches.length > 0 && context.branches
       .filter(branch => !branch.remote)

libs/remix-ui/git/src/components/panels/remotesimport.tsx

@@ -6,6 +6,7 @@ import { gitPluginContext } from "../gitui";
 import Select from 'react-select'
 import { selectStyles, selectTheme } from "../../types/styles";
 import { TokenWarning } from "./tokenWarning";
+import RepositorySelect from "../github/repositoryselect";
 
 
 export const RemotesImport = () => {
@@ -18,7 +19,6 @@ export const RemotesImport = () => {
   const [remoteName, setRemoteName] = useState('')
 
   useEffect(() => {
-    console.log('context', context.repositories)
     if (context.repositories && context.repositories.length > 0) {
       // map context.repositories to options
       const options = context.repositories && context.repositories.length > 0 && context.repositories.map(repo => {
@@ -44,19 +44,15 @@ export const RemotesImport = () => {
     }
   };
 
-  const selectRepo = async (value: number | string) => {
+  const selectRepo = async (repo: repository) => {
-    // find repo
+    setRepo(repo)
-    console.log('setRepo', value, context.repositories)
-
-    const repo = context.repositories.find(repo => {
-      return repo.id.toString() === value.toString()
-    })
-    console.log('repo', repo)
-    if (repo) {
-      setRepo(repo)
-    }
   }
 
+
+  useEffect(() => {
+    console.log('OTIONS', repoOtions)
+  },[repoOtions])
+
   const addRemote = async () => {
     try {
       actions.addRemote({
@@ -75,20 +71,7 @@ export const RemotesImport = () => {
   return (
     <>
       <TokenWarning />
-      <Button onClick={fetchRepositories} className="w-100 mt-1">
+      <RepositorySelect select={selectRepo} />
-        <i className="fab fa-github mr-1"></i>Fetch Remotes from GitHub
-      </Button>
-      {show ?
-        <Select
-          options={repoOtions}
-          className="mt-1"
-          onChange={(e: any) => e && selectRepo(e.value)}
-          theme={selectTheme}
-          styles={selectStyles}
-          isClearable={true}
-          placeholder="Type to search for a repository..."
-          isLoading={loading}
-        /> : null}
 
       {repo ?
         <input placeholder="remote name" name='remotename' onChange={e => onRemoteNameChange(e.target.value)} value={remoteName} className="form-control mb-2" type="text" id="remotename" />

libs/remix-ui/git/src/components/panels/repositories.tsx

@@ -1,125 +0,0 @@
-import React, { useEffect, useState } from "react";
-import { Alert, Button } from "react-bootstrap";
-import { gitActionsContext } from "../../state/context";
-import { repository } from "../../types";
-import { gitPluginContext } from "../gitui";
-import AsyncSelect from 'react-select'
-import { selectStyles, selectTheme } from "../../types/styles";
-import { TokenWarning } from "./tokenWarning";
-import RepositorySelect from "../github/repositoryselect";
-
-interface RepositoriesProps {
-  cloneDepth?: number
-  cloneAllBranches?: boolean
-}
-
-export const Repositories = (props: RepositoriesProps) => {
-  const { cloneDepth, cloneAllBranches } = props
-  const context = React.useContext(gitPluginContext)
-  const actions = React.useContext(gitActionsContext)
-  const [branch, setBranch] = useState({ name: "" });
-  const [repo, setRepo] = useState<repository>(null);
-  const [repoOtions, setRepoOptions] = useState<any>([]);
-  const [branchOptions, setBranchOptions] = useState<any>([]);
-  const [loading, setLoading] = useState(false)
-  const [show, setShow] = useState(false)
-
-  useEffect(() => {
-    console.log('context', context.repositories)
-    if (context.repositories && context.repositories.length > 0) {
-      // map context.repositories to options
-      const options = context.repositories && context.repositories.length > 0 && context.repositories.map(repo => {
-        return { value: repo.id, label: repo.full_name }
-      })
-      setRepoOptions(options)
-    } else {
-      setRepoOptions(null)
-      setShow(false)
-    }
-    setLoading(false)
-
-  }, [context.repositories])
-
-
-  useEffect(() => {
-    // map context.branches to options
-    const options = context.remoteBranches && context.remoteBranches.length > 0 && context.remoteBranches.map(branch => {
-      return { value: branch.name, label: branch.name }
-    }
-    )
-    setBranchOptions(options)
-  }, [context.remoteBranches])
-
-
-  useEffect(() => {
-    console.log('show', show)
-  },[show])
-
-  const fetchRepositories = async () => {
-    try {
-      setShow(true)
-      setLoading(true)
-      setRepoOptions([])
-      setBranchOptions([])
-      console.log(await actions.repositories())
-    } catch (e) {
-      // do nothing
-    }
-  };
-
-  const selectRepo = async (value: number | string) => {
-    // find repo
-    console.log('setRepo', value, context.repositories)
-
-    const repo = context.repositories.find(repo => {
-      return repo.id.toString() === value.toString()
-    })
-    console.log('repo', repo)
-    if (repo) {
-      setBranchOptions([])
-      setBranch({ name: "" })
-      setRepo(repo)
-      await actions.remoteBranches(repo.owner.login, repo.name)
-    }
-  }
-
-  const selectRemoteBranch = async (value: number | string) => {
-    console.log('setRemoteBranch', value)
-    setBranch({ name: value.toString() })
-  }
-
-  const clone = async () => {
-    try {
-      console.log('clone', repo, branch)
-      actions.clone(repo.html_url, branch.name, cloneDepth, !cloneAllBranches)
-    } catch (e) {
-      // do nothing
-    }
-  };
-
-  
-  return (
-    <>
-      <RepositorySelect />
-
-      {branchOptions && branchOptions.length ?
-        <AsyncSelect
-          options={branchOptions}
-          className="mt-1"
-          onChange={(e: any) => e && selectRemoteBranch(e.value)}
-          theme={selectTheme}
-          styles={selectStyles}
-          isClearable={true}
-          placeholder="Type to search for a branch..."
-        /> : null}
-
-      {repo && branch.name && branch.name !== '0' ?
-        <button data-id='clonebtn' className='btn btn-primary mt-1 w-100' onClick={async () => {
-          await clone()
-        }}>clone {repo.full_name}:{branch.name}</button> : null}
-
-    </>
-  )
-}
-
-

libs/remix-ui/git/src/lib/gitactions.ts

@@ -470,8 +470,21 @@ export const repositories = async () => {
   try {
     const token = await tokenWarning();
     if (token) {
-      const repos = await plugin.call('dGitProvider' as any, 'repositories', { token });
+      let repos = await plugin.call('dGitProvider' as any, 'repositories', { token, per_page: 100 })
       dispatch(setRepos(repos))
+      let page = 2
+      let hasMoreData = true
+      const per_page = 100
+      while(hasMoreData){
+        let pagedResponse = await plugin.call('dGitProvider' as any, 'repositories', { token, page:page, per_page: per_page })
+        if(pagedResponse.length < per_page){
+          hasMoreData = false
+        }
+        repos = [...repos,...pagedResponse]
+        dispatch(setRepos(repos))
+        page++
+      }
+
     } else {
       plugin.call('notification', 'alert', {
         title: 'Error getting repositories',
@@ -493,8 +506,20 @@ export const remoteBranches = async (owner: string, repo: string) => {
   try {
     const token = await tokenWarning();
     if (token) {
-      const branches = await plugin.call('dGitProvider' as any, 'remotebranches', { token, owner, repo });
+      let branches = await plugin.call('dGitProvider' as any, 'remotebranches', { token, owner, repo, per_page: 100 });
       dispatch(setRemoteBranches(branches))
+      let page = 2
+      let hasMoreData = true
+      const per_page = 100
+      while(hasMoreData){
+        let pagedResponse = await plugin.call('dGitProvider' as any, 'remotebranches', { token, owner, repo, page:page, per_page: per_page })
+        if(pagedResponse.length < per_page){
+          hasMoreData = false
+        }
+        branches = [...branches,...pagedResponse]
+        dispatch(setRemoteBranches(branches))
+        page++
+      }
     } else {
       plugin.call('notification', 'alert', {
         title: 'Error getting branches',