diff --git a/fetchRepos.js b/fetchRepos.js index e61499cc5..1858fb9be 100755 --- a/fetchRepos.js +++ b/fetchRepos.js @@ -13,9 +13,8 @@ const DbFile = require('./impl/dbFile'); const fetchJson = require('./impl/fetchJson'); - const ghAPIVersion = process.env.GHUSER_GHV4 === "true" ? 4 : 3 - console.log(`Using GitHub APIv${ghAPIVersion}`) - const ghcl = require(`./impl/githubV${ghAPIVersion}`); + const ghclV3 = require(`./impl/githubV3`); + const ghclV4 = require(`./impl/githubV4`); const githubColors = require('github-colors'); const scriptUtils = require('./impl/scriptUtils'); @@ -47,10 +46,7 @@ optional arguments: return; async function fetchRepos(firsttime) { - let spinner; - - let spinnerText = 'Reading users from DB...'; - spinner = ora(spinnerText).start(); + console.log('Reading users from DB...') const users = []; for (const file of fs.readdirSync(data.users)) { await sleep(0); // make loop interruptible @@ -59,14 +55,12 @@ optional arguments: const user = new DbFile(path.join(data.users, file)); if (!user.ghuser_deleted_because && !user.removed_from_github) { users.push(user); - spinner.text = `${spinnerText} [${users.length}]`; } } } - spinner.succeed(`Found ${users.length} users in DB`); + console.log(`Found ${users.length} users in DB`); - spinnerText = 'Searching repos referenced by users...'; - spinner = ora(spinnerText).start(); + console.log('Searching repos referenced by users...'); const referencedRepos = new Set([]); for (const user of users) { for (const repo in (user.contribs && user.contribs.repos || [])) { @@ -77,8 +71,6 @@ optional arguments: throw `user.contribs.repos[${repo}] is undefined`; } referencedRepos.add(full_name); - spinner.text = `${spinnerText} [${referencedRepos.size}]`; - // Make sure the corresponding repo files exist: for (const pathToFolder of [data.repos, data.repoCommits]) { const filePath = path.join(pathToFolder, `${full_name}.json`); @@ -86,10 +78,9 @@ optional arguments: } } } - spinner.succeed(`Found ${referencedRepos.size} repos referenced by users`); + console.log(`Found ${referencedRepos.size} repos referenced by users`); - spinnerText = 'Reading repos from DB...'; - spinner = ora(spinnerText).start(); + console.log('Reading repos from DB...'); const reposInDb = new Set([]); for (const ownerDir of fs.readdirSync(data.repos)) { const pathToOwner = path.join(data.repos, ownerDir); @@ -100,66 +91,76 @@ optional arguments: if (file.endsWith(ext)) { const fullName = `${ownerDir}/${file}`.slice(0, -ext.length); reposInDb.add(fullName); - spinner.text = `${spinnerText} [${reposInDb.size}]`; } } } - spinner.succeed(`Found ${reposInDb.size} repos in DB`); + console.log(`Found ${Object.keys(repoPaths).length} repos in DB`); - for (const repoFullName of referencedRepos) { - await fetchRepo(repoFullName, firsttime); - } stripUnreferencedRepos(); - for (const repoFullName of reposInDb) { - const repo = new DbFile(repoPaths(repoFullName).repo); + let full_names; + + full_names = [...referencedRepos]; + await Promise.all([ + loopDo(full_names, async (full_name) => { await fetchRepo(ghclV3, full_name, firsttime)}), + loopDo(full_names, async (full_name) => { await fetchRepo(ghclV4, full_name, firsttime)}), + ]); + + full_names = [...reposInDb]; + await Promise.all([ + loopDo(full_names, async (full_name) => { await fetchRepoDetails(ghclV3, repoPaths(full_name))}), + loopDo(full_names, async (full_name) => { await fetchRepoDetails(ghclV4, repoPaths(full_name))}), + ]); + + createRenamedRepos(); + + return; + + async function fetchRepoDetails(ghcl, paths) { + const repo = new DbFile(paths.repo); if (!repo.removed_from_github && !repo.ghuser_insignificant) { - const repoCommits = new DbFile(repoPaths(repoFullName).repoCommits); - await fetchRepoCommitsAndContributors(repo, repoCommits); - await fetchRepoPullRequests(repo); - if (ghAPIVersion === 3) { - await fetchRepoLanguages(repo); - } + const repoCommits = new DbFile(paths.repoCommits); + await fetchRepoCommitsAndContributors(ghcl, repo, repoCommits); + await fetchRepoPullRequests(ghcl, repo); + await fetchRepoLanguages(ghcl, repo); await fetchRepoSettings(repo); } markRepoAsFullyFetched(repo); } - createRenamedRepos(); + async function fetchRepo(ghcl, repoFullName, firsttime) { + const tag = `[${ghcl.version}] Fetch Repo - ${repoFullName} -`; - return; - - async function fetchRepo(repoFullName, firsttime) { - spinner = ora(`Fetching ${repoFullName}...`).start(); + console.log(`${tag} starting`); const repo = new DbFile(repoPaths(repoFullName).repo); const now = new Date; const maxAgeHours = firsttime && (24 * 365) || 12; if (repo.fetching_since || repo.fetched_at && now - Date.parse(repo.fetched_at) < maxAgeHours * 60 * 60 * 1000) { - spinner.succeed(`${repoFullName} is still fresh`); + console.log(`${tag} is still fresh`); return; } if (repo.removed_from_github) { // For now ok, but maybe some day we'll have to deal with resurrected repos. - spinner.succeed(`${repoFullName} was removed from GitHub in the past`); + console.log(`${tag} was removed from GitHub in the past`); return; } - const ghDataJson = await ghcl.repo(spinner, [304, 403, 404, 451], repoFullName, + const ghDataJson = await ghcl.repo([304, 403, 404, 451], repoFullName, new Date(repo.fetched_at)); switch (ghDataJson) { case 304: - repo.fetched_at = now.toISOString();; - spinner.succeed(`${repoFullName} didn't change`); + repo.fetched_at = now.toISOString(); + console.log(`${tag} didn't change`); repo.write(); return; case 404: repo.removed_from_github = true; - spinner.succeed(`${repoFullName} was removed from GitHub`); + console.log(`${tag} was removed from GitHub`); repo.write(); return; @@ -167,14 +168,14 @@ optional arguments: case 451: // Probably a DCMA takedown, like https://github.com/worktips/worktips case 403: // Probably not respecting the Terms of Service, like https://github.com/Kwoth/NadekoBot repo.removed_from_github = true; - spinner.succeed(`${repoFullName} is blocked for legal reasons`); + console.log(`${tag} is blocked for legal reasons`); repo.write(); return; } - repo.fetching_since = now.toISOString();; + repo.fetching_since = now.toISOString(); - spinner.succeed(`Fetched ${repoFullName}`); + console.log(`${tag} finished`); ghDataJson.owner = ghDataJson.owner.login; Object.assign(repo, ghDataJson); @@ -224,7 +225,9 @@ optional arguments: } } - async function fetchRepoCommitsAndContributors(repo, repoCommits) { + async function fetchRepoCommitsAndContributors(ghcl, repo, repoCommits) { + const tag = `[${ghcl.version}] Fetch Commits & Contribs - ${repo.full_name} -`; + repo.contributors = repo.contributors || {}; repoCommits.contributors = repoCommits.contributors || {}; repoCommits.last_fetched_commit = repoCommits.last_fetched_commit || { @@ -232,8 +235,7 @@ optional arguments: date: '2000-01-01T00:00:00Z' }; - const spinnerText = `Fetching ${repo.full_name}'s commits...`; - spinner = ora(spinnerText).start(); + console.log(`${tag} starting`); if (!repo.fetching_since || repo.fetched_at && new Date(repo.fetched_at) > new Date(repo.pushed_at)) { @@ -242,7 +244,7 @@ optional arguments: // This is fishy because the repo isn't empty but we didn't manage to fetch any commit // last time. We definitely need to try again. } else { - spinner.succeed(`${repo.full_name} hasn't changed`); + console.log(`${tag} hasn't changed`); return; } @@ -254,15 +256,15 @@ optional arguments: const perPage = 100; pages: for (let page = 1;; ++page) { - spinner.start(`${spinnerText} [page ${page}]`); - const ghDataJson = await ghcl.commits(spinner, [404, 500], repo.full_name, repoCommits.last_fetched_commit.date, page, perPage, ghAPIV4Cursor); - ghAPIV4Cursor = ghDataJson[0] ? ghDataJson[0].cursor : undefined + console.log(`${tag} [page ${page}]`); + const ghDataJson = await ghcl.commits([404, 500], repo.full_name, repoCommits.last_fetched_commit.date, page, perPage, ghAPIV4Cursor); + ghAPIV4Cursor = ghDataJson[0] ? ghDataJson[0].cursor : undefined; switch (ghDataJson) { case 404: // The repo has been removed during the current run. It will be marked as removed in the // next run. For now just don't crash. - spinner.succeed(`${repo.full_name} was just removed from GitHub`); + console.log(`${tag} was just removed from GitHub`); return; case 500: // Workaround for #8 if (page > 1000) { @@ -270,7 +272,7 @@ optional arguments: repoCommits.ghuser_truncated = true; break pages; } - spinner.fail(); + console.log(`${tag} failed`); return; } @@ -332,12 +334,12 @@ optional arguments: } if (page >= 10000) { - spinner.fail(); + console.log(`${tag} failed`); throw 'fetchRepoCommitsAndContributors(): Infinite loop?'; } } - spinner.succeed(`Fetched ${repo.full_name}'s commits`); + console.log(`${tag} finished`); repo.write(); repoCommits.last_fetched_commit = mostRecentCommit || repoCommits.last_fetched_commit; @@ -348,12 +350,14 @@ optional arguments: } } - async function fetchRepoPullRequests(repo) { - spinner = ora(`Fetching ${repo.full_name}'s pull requests...`).start(); + async function fetchRepoPullRequests(ghcl, repo) { + const tag = `[${ghcl.version}] Fetch PRs - ${repo.full_name} -`; + + console.log(`${tag} starting`); if (!repo.fetching_since || repo.fetched_at && new Date(repo.fetched_at) > new Date(repo.pushed_at)) { - spinner.succeed(`${repo.full_name} hasn't changed`); + console.log(`${tag} hasn't changed`); return; } @@ -362,22 +366,22 @@ optional arguments: let ghAPIV4Cursor; const perPage = 100; for (let page = 1;; ++page) { - const ghDataJson = await ghcl.pullRequests(spinner, [404, 500, 502], repo.full_name, page, - perPage, ghAPIV4Cursor); + const ghDataJson = await ghcl.pullRequests([404, 500, 502], repo.full_name, page, perPage, + ghAPIV4Cursor); ghAPIV4Cursor = ghDataJson[0] ? ghDataJson[0].cursor : undefined; switch (ghDataJson) { case 404: // The repo has been removed during the current run. It will be marked as removed in the // next run. For now just don't crash. - spinner.succeed(`${repo.full_name} was just removed from GitHub`); + console.log(`${tag} was just removed from GitHub`); return; case 502: // About twice per month we hit 502 'Server Error', often on repo // everypolitician/everypolitician-data, so let's not crash and move on, even if this // means that we'll miss some pull requests sometimes. case 500: // Workaround for #8 - spinner.fail(); + console.log(`${tag} failed`); return; } @@ -397,35 +401,37 @@ optional arguments: } if (page >= 10000) { - spinner.fail(); + console.log(`${tag} failed`); throw 'fetchRepoPullRequests(): Infinite loop?'; } } - spinner.succeed(`Fetched ${repo.full_name}'s pull requests`); + console.log(`${tag} finished`); repo.pulls_authors = [...authors]; repo.write(); } - async function fetchRepoLanguages(repo) { - spinner = ora(`Fetching ${repo.full_name}'s languages...`).start(); + async function fetchRepoLanguages(ghcl, repo) { + const tag = `[${ghcl.version}] Fetch Languages - ${repo.full_name} -`; + + console.log(`${tag} starting`); if (!repo.fetching_since || repo.fetched_at && new Date(repo.fetched_at) > new Date(repo.pushed_at)) { - spinner.succeed(`${repo.full_name} hasn't changed`); + console.log(`${tag} hasn't changed`); return; } - const ghDataJson = await ghcl.repoLanguages(spinner, [404], repo.full_name); + const ghDataJson = await ghcl.repoLanguages([404], repo.full_name); if (ghDataJson === 404) { // The repo has been removed during the current run. It will be marked as removed in the // next run. For now just don't crash. - spinner.succeed(`${repo.full_name} was just removed from GitHub`); + console.log(`${tag} was just removed from GitHub`); return; } - spinner.succeed(`Fetched ${repo.full_name}'s languages`); + console.log(`${tag} finished`); for (let language in ghDataJson) { ghDataJson[language] = { @@ -439,28 +445,30 @@ optional arguments: } async function fetchRepoSettings(repo) { - spinner = ora(`Fetching ${repo.full_name}'s settings...`).start(); + const tag = `Fetch Settings - ${repo.full_name} -`; + + console.log(`${tag} starting`); if (!repo.fetching_since || repo.fetched_at && new Date(repo.fetched_at) > new Date(repo.pushed_at)) { - spinner.succeed(`${repo.full_name} hasn't changed`); + console.log(`${tag} hasn't changed`); return; } for (const fileName of ['.ghuser.io.json', '.github/ghuser.io.json']) { const url = `https://raw.githubusercontent.com/${repo.full_name}/master/${fileName}`; - const dataJson = await fetchJson(url, spinner, [404]); + const dataJson = await fetchJson(url, null, [404]); if (dataJson === 404) { continue; } - spinner.succeed(`Fetched ${repo.full_name}'s settings`); + console.log(`${tag} finished`); repo.settings = dataJson; repo.write(); return; } - spinner.succeed(`${repo.full_name} has no settings`); + console.log(`${tag} has no settings`); } function markRepoAsFullyFetched(repo) { @@ -508,3 +516,15 @@ optional arguments: } })(); + +const loopDo = async (jobs, jobFn) => { + while(true) { + const job = jobs.pop(); + + if (!job) { + break + } + + await jobFn(job); + } +}; diff --git a/impl/data.js b/impl/data.js index 36f1c7999..7614bfd04 100755 --- a/impl/data.js +++ b/impl/data.js @@ -8,7 +8,7 @@ const dbPath = process.env.GHUSER_DBDIR || path.join(os.homedir(), 'data'); if (!fs.existsSync(dbPath)) { - throw `${dbPath} directory doesn't exist` + throw `${dbPath} directory doesn't exist`; } module.exports = { diff --git a/impl/github.js b/impl/github.js index 334847d68..fb3cb0748 100755 --- a/impl/github.js +++ b/impl/github.js @@ -49,12 +49,12 @@ // Waits until we are far away from hitting GitHub's API rate limit. async function waitForRateLimit(oraSpinner, isGraphQL) { const oldSpinnerText = oraSpinner && oraSpinner.text; + const key = isGraphQL ? "graphql" : "core"; let rateLimit = await fetchGHRateLimit(oraSpinner); - const lim = rateLimit[isGraphQL ? "graphql" : "core"]; - if (lim.remaining <= 10) { + if (rateLimit[key].remaining <= 10) { const now = (new Date).getTime() / 1000; - const secondsToSleep = Math.ceil(lim.reset - now) + 1; + const secondsToSleep = Math.ceil(rateLimit[key].reset - now) + 1; if (secondsToSleep >= 0) { if (oraSpinner) { oraSpinner.text += ` (waiting ${secondsToSleep} second(s) for API rate limit)`; diff --git a/impl/githubV3.js b/impl/githubV3.js index 869145dd3..6980700c7 100644 --- a/impl/githubV3.js +++ b/impl/githubV3.js @@ -5,27 +5,30 @@ const gh = require('./github'); - const repo = async (oraSpinner, errCodes, repoFullName, repoFetchedAtDate) => { + const version = "V3"; + + const repo = async (errCodes, repoFullName, repoFetchedAtDate) => { const ghRepoUrl = `https://api.github.com/repos/${repoFullName}`; - return await gh.fetchGHJson(ghRepoUrl, oraSpinner, errCodes, repoFetchedAtDate); + return await gh.fetchGHJson(ghRepoUrl, null, errCodes, repoFetchedAtDate); }; - const commits = async (oraSpinner, errCodes, repoFullName, lastFetchedCommitDateStr, page, perPage) => { + const commits = async (errCodes, repoFullName, lastFetchedCommitDateStr, page, perPage) => { const ghUrl = `https://api.github.com/repos/${repoFullName}/commits?since=${lastFetchedCommitDateStr}&page=${page}&per_page=${perPage}`; - return await gh.fetchGHJson(ghUrl, oraSpinner, errCodes); - } + return await gh.fetchGHJson(ghUrl, null, errCodes); + }; - const pullRequests = async (oraSpinner, errCodes, repoFullName, page, perPage) => { + const pullRequests = async (errCodes, repoFullName, page, perPage) => { const ghUrl = `https://api.github.com/repos/${repoFullName}/pulls?state=all&page=${page}&per_page=${perPage}`; - return await gh.fetchGHJson(ghUrl, oraSpinner, errCodes); - } + return await gh.fetchGHJson(ghUrl, null, errCodes); + }; - const repoLanguages = async (oraSpinner, errCodes, repoFullName) => { + const repoLanguages = async (errCodes, repoFullName) => { const ghUrl = `https://api.github.com/repos/${repoFullName}/languages`; - return await gh.fetchGHJson(ghUrl, oraSpinner, errCodes); - } + return await gh.fetchGHJson(ghUrl, null, errCodes); + }; module.exports = { + version, repo, commits, pullRequests, diff --git a/impl/githubV4.js b/impl/githubV4.js index b73cccf9e..cd997bf86 100644 --- a/impl/githubV4.js +++ b/impl/githubV4.js @@ -6,6 +6,8 @@ const gh = require('./github'); + const version = "V4"; + const repoQuery = ` query($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { @@ -14,6 +16,7 @@ query($owner: String!, $name: String!) { isPrivate owner { login + __typename } url description @@ -41,78 +44,64 @@ query($owner: String!, $name: String!) { defaultBranchRef { name } - languages(first: 100) { - edges { - size - node { - color - name - } - } - } } } -` - const repo = async (oraSpinner, errCodes, repoFullName, repoFetchedAtDate) => { - // TODO use repoFetchedAtDate +`; + // TODO: This sets "organization" field if appropriate, but as an empty object. + const repo = async (errCodes, repoFullName, repoFetchedAtDate) => { + // TODO: Use repoFetchedAtDate - const dataJson = await gh.fetchGHJson('https://api.github.com/graphql', oraSpinner, errCodes, null, { + const dataJson = await gh.fetchGHJson('https://api.github.com/graphql', null, errCodes, null, { query: repoQuery, variables: buildCommonRepoVariables(repoFullName), - }) + }); - if (!(dataJson instanceof Object)) { - return dataJson + const err = checkResponse(dataJson); + if (err != null) { + return err } - if (dataJson.errors) { - switch (dataJson.errors[0].type) { - case "NOT_FOUND": - return 404 - } - } - - const r = dataJson.data.repository - let res = {} - res.name = r.name - res.full_name = r.nameWithOwner - res.private = r.isPrivate - res.owner = r.owner.login - res.html_url = r.url - res.description = r.description - res.fork = r.isFork - res.url = "https://api.github.com/repos/" + r.nameWithOwner - res.languages_url = "https://api.github.com/repos/" + r.nameWithOwner + "/languages" - res.pulls_url = "https://api.github.com/repos/" + r.nameWithOwner + "/pulls{/number}" + const r = dataJson.data.repository; + + let res = {}; + res.name = r.name; + res.full_name = r.nameWithOwner; + res.private = r.isPrivate; + res.owner = { + login: r.owner.login + }; + res.html_url = r.url; + res.description = r.description; + res.fork = r.isFork; + res.url = "https://api.github.com/repos/" + r.nameWithOwner; + res.languages_url = "https://api.github.com/repos/" + r.nameWithOwner + "/languages"; + res.pulls_url = "https://api.github.com/repos/" + r.nameWithOwner + "/pulls{/number}"; // format: "2015-09-10T02:15:47Z" - res.created_at = coerceDate(r.createdAt) - res.updated_at = coerceDate(r.updatedAt) - res.pusher_at = coerceDate(r.pushedAt) - - res.homepage = r.homepageUrl // TODO verify expected URL - res.size = r.diskUsage - res.stargazers_count = r.stargazers.totalCount - res.language = r.primaryLanguage.name - res.mirror_url = r.mirrorUrl - res.archived = r.isArchived - res.default_branch = r.defaultBranchRef.name + res.created_at = coerceDate(r.createdAt); + res.updated_at = coerceDate(r.updatedAt); + res.pushed_at = coerceDate(r.pushedAt); + + res.homepage = r.homepageUrl; // TODO verify expected URL + res.size = r.diskUsage; + res.stargazers_count = r.stargazers.totalCount; + res.language = r.primaryLanguage ? r.primaryLanguage.name : null; + res.mirror_url = r.mirrorUrl; + res.archived = r.isArchived; if (r.licenseInfo) { - res.license = {} - res.license.key = r.licenseInfo.key - res.license.name = r.licenseInfo.name - res.license.spdx_id = r.licenseInfo.spdxId - res.license.url = r.licenseInfo.url - res.license.node_id = r.licenseInfo.id + res.license = {}; + res.license.key = r.licenseInfo.key; + res.license.name = r.licenseInfo.name; + res.license.spdx_id = r.licenseInfo.spdxId; + res.license.url = r.licenseInfo.url; + res.license.node_id = r.licenseInfo.id; } - res.languages = {} - for (let it of r.languages.edges) { - res.languages[it.node.name] = { - bytes: it.size, - color: it.node.color, - } + res.default_branch = r.defaultBranchRef.name; + + if (r.owner.__typename === "Organization") { + res.organization = {}; } return res @@ -149,26 +138,26 @@ query($owner: String!, $name: String!, $cursor: String, $since: GitTimestamp) { } } } -` - const commits = async (oraSpinner, errCodes, repoFullName, lastFetchedCommitDateStr, page, perPage, v4cursor = null) => { +`; + const commits = async (errCodes, repoFullName, lastFetchedCommitDateStr, page, perPage, v4cursor = null) => { - let variables = buildCommonRepoVariables(repoFullName, page, v4cursor) - variables.since = lastFetchedCommitDateStr - const dataJson = await gh.fetchGHJson('https://api.github.com/graphql', oraSpinner, errCodes, null, { + let variables = buildCommonRepoVariables(repoFullName, page, v4cursor); + variables.since = lastFetchedCommitDateStr; + const dataJson = await gh.fetchGHJson('https://api.github.com/graphql', null, errCodes, null, { query: commitsQuery, variables: variables, - }) + }); if (!(dataJson instanceof Object)) { return dataJson } - let edges = dataJson.data.repository.ref.target.history.edges + let edges = dataJson.data.repository.ref.target.history.edges; - let res = [] + let res = []; for (let e of edges) { - const author = e.node.author - const committer = e.node.committer + const author = e.node.author; + const committer = e.node.committer; res.push({ sha: e.node.oid, @@ -189,7 +178,7 @@ query($owner: String!, $name: String!, $cursor: String, $since: GitTimestamp) { }) } - insertCursor(res, edges) + insertCursor(res, edges); return res }; @@ -210,21 +199,21 @@ query($owner: String!, $name: String!, $cursor: String) { } } } -` - const pullRequests = async (oraSpinner, errCodes, repoFullName, page, perPage, v4cursor = null) => { +`; + const pullRequests = async (errCodes, repoFullName, page, perPage, v4cursor = null) => { - const dataJson = await gh.fetchGHJson('https://api.github.com/graphql', oraSpinner, errCodes, null, { + const dataJson = await gh.fetchGHJson('https://api.github.com/graphql', null, errCodes, null, { query: pullRequestsQuery, variables: buildCommonRepoVariables(repoFullName, page, v4cursor), - }) + }); if (!(dataJson instanceof Object)) { return dataJson } - let edges = dataJson.data.repository.pullRequests.edges + let edges = dataJson.data.repository.pullRequests.edges; - let res = [] + let res = []; for (let e of edges) { res.push({ user: { @@ -233,31 +222,64 @@ query($owner: String!, $name: String!, $cursor: String) { }) } - insertCursor(res, edges) + insertCursor(res, edges); return res - } + }; - const repoLanguages = async () => { - throw "unexpected call to repoLanguages" - return + const repoLanguagesQuery = ` +query($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + languages(first: 100) { + edges { + size + node { + name + } + } + } } +} +`; + const repoLanguages = async (errCodes, repoFullName) => { + + const dataJson = await gh.fetchGHJson('https://api.github.com/graphql', null, errCodes, null, { + query: repoLanguagesQuery, + variables: buildCommonRepoVariables(repoFullName), + }); + + const err = checkResponse(dataJson); + if (err != null) { + return err + } + + const r = dataJson.data.repository; + + let res = {}; + + for (let it of r.languages.edges) { + res[it.node.name] = it.size + } + + return res + }; module.exports = { + version, repo, commits, pullRequests, repoLanguages, - } + }; function buildCommonRepoVariables(repoFullName, page, cursor) { - let owner, repoName - [owner, repoName] = repoFullName.split("/") + let owner, repoName; + [owner, repoName] = repoFullName.split("/"); let variables = { owner: owner, name: repoName, - } + }; if (isNaN(page) || page === 1) { return variables @@ -267,7 +289,7 @@ query($owner: String!, $name: String!, $cursor: String) { throw "expected cursor not null" } - variables.cursor = cursor + variables.cursor = cursor; return variables } @@ -276,8 +298,7 @@ query($owner: String!, $name: String!, $cursor: String) { // Pass the cursor in the first element of the response. function insertCursor(resultArray, edgesArray) { if (resultArray.length > 0 && edgesArray.length > 0) { - const cursor = edgesArray.slice(-1)[0].cursor - console.log(cursor) + const cursor = edgesArray.slice(-1)[0].cursor; resultArray[0].cursor = cursor } } @@ -289,4 +310,19 @@ query($owner: String!, $name: String!, $cursor: String) { return (new Date(dateStr)).toISOString() } + function checkResponse(dataJson) { + if (!(dataJson instanceof Object)) { + return 500 + } + if (dataJson.errors) { + switch (dataJson.errors[0].type) { + case "NOT_FOUND": + return 404; + default: + return 500; + } + } + return null; + } + })();