Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: fetch all repos for top languages #1178

Closed
wants to merge 2 commits into from
Closed

fix: fetch all repos for top languages #1178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
564a1a2
fix: fetch all repos for top languages
tc-imba Jul 8, 2021
0b90293
fix: fetch all repos for top languages (unify the variable names)
tc-imba Jul 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions src/fetchers/top-languages-fetcher.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,16 @@ const fetcher = (variables, token) => {
return request(
{
query: `
query userInfo($login: String!) {
query userInfo($login: String!, $first: Int!, $after: String) {
user(login: $login) {
# fetch only owner repos & not forks
repositories(ownerAffiliations: OWNER, isFork: false, first: 100) {
repositories(
ownerAffiliations: OWNER, isFork: false,
first: $first, after: $after
) {
edges {
cursor
}
nodes {
name
languages(first: 10, orderBy: {field: SIZE, direction: DESC}) {
Expand Down Expand Up @@ -37,14 +43,28 @@ const fetcher = (variables, token) => {
async function fetchTopLanguages(username, exclude_repo = []) {
if (!username) throw Error("Invalid username");

const res = await retryer(fetcher, { login: username });
const pageSize = 100;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we can use the pageInfo object to get cleaner code. 🚀

  // Loop through all pages and fetch all repos
  let repoNodes = [];
  let hasNextPage = true
  let endCursor = null
  while (hasNextPage) {
      const variables = { login: username, first: 100, after: endCursor };
      const res = await retryer(fetcher, variables);
      
      if (res.data.errors) {
        logger.error(res.data.errors);
        throw Error(res.data.errors[0].message || "Could not fetch user");
      }
      
      repoNodes.push(...res.data.data.user.repositories.nodes);
      hasNextPage = res.data.data.user.repositories.pageInfo.hasNextPage
      endCursor = res.data.data.user.repositories.pageInfo.endCursor
  }

let pageCursor = null;

let repoNodes = [];

while (true) {
const variables = { login: username, first: pageSize, after: pageCursor };
const res = await retryer(fetcher, variables);

if (res.data.errors) {
logger.error(res.data.errors);
throw Error(res.data.errors[0].message || "Could not fetch user");
}

if (res.data.errors) {
logger.error(res.data.errors);
throw Error(res.data.errors[0].message || "Could not fetch user");
repoNodes = repoNodes.concat(res.data.data.user.repositories.nodes);
if (!res.data.data.user.repositories.edges ||
res.data.data.user.repositories.edges.length < pageSize) {
break;
}
pageCursor = res.data.data.user.repositories.edges[pageSize - 1].cursor;
}

let repoNodes = res.data.data.user.repositories.nodes;
let repoToHide = {};

// populate repoToHide map for quick lookup
Expand Down
6 changes: 6 additions & 0 deletions tests/fetchTopLanguages.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ const data_langs = {
data: {
user: {
repositories: {
edges: [
{ cursor: "1" },
{ cursor: "2" },
{ cursor: "3" },
{ cursor: "4" },
],
nodes: [
{
languages: {
Expand Down