From 305d16cc3a2b954132b2ee9ec604bbaf3ab99e70 Mon Sep 17 00:00:00 2001 From: omkar Date: Tue, 3 Dec 2019 22:29:43 +0530 Subject: [PATCH 1/3] comments --- src/CacheService.js | 3 ++- src/ContributorRepository.js | 6 ++++++ src/ContributorService.js | 13 +++++++++++-- src/Routes.js | 4 ++++ src/settings.js | 4 +++- 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/CacheService.js b/src/CacheService.js index ee55132..4c49231 100644 --- a/src/CacheService.js +++ b/src/CacheService.js @@ -7,12 +7,13 @@ class Cache { checkperiod: ttlSeconds * 0.2, }); } + // get contributors from cache get(key, gitHubApi) { const value = this.cache.get(key); if (value) { return Promise.resolve(value); } - + // or get it from github return gitHubApi.then((result) => { this.cache.set(key, result); return result; diff --git a/src/ContributorRepository.js b/src/ContributorRepository.js index e22756f..b270128 100644 --- a/src/ContributorRepository.js +++ b/src/ContributorRepository.js @@ -1,3 +1,6 @@ +/* + Lower level utility methods to fetch data from github and preprocess that data +*/ const octokit = require("@octokit/rest"); const githubApi = new octokit(); const async = require('async'); @@ -30,6 +33,7 @@ function getContributorsByRepo(org, repositoryName) { "User-Agent": settings.organisation } }); + // collect all contributors for a repo, map them to Profile class githubApi.paginate(options) .then(data => { data = data @@ -54,6 +58,7 @@ function getReposByOrg(organisation) { "User-Agent": settings.organisation } }); + // collect all repos for an organisation, map the data to Repositor class and return githubApi.paginate(options) .then(data => { data = data @@ -66,6 +71,7 @@ function getReposByOrg(organisation) { }); }); } +// Flatten 2D array consisting of unique users function preProcess(data) { return new Promise(async (resolve, reject) => { data = Array.prototype.concat.apply([], data); diff --git a/src/ContributorService.js b/src/ContributorService.js index 595e7e0..1dde2fd 100644 --- a/src/ContributorService.js +++ b/src/ContributorService.js @@ -1,10 +1,14 @@ +/* + Middle level service methods which interact with CacheService and ContributorRepository +*/ const ContributorRepository = require('./ContributorRepository'); -const contributors = ContributorRepository.getContributorsByOrg; +const getContributors = ContributorRepository.getContributorsByOrg; const CacheService = require('./CacheService.js'); const settings = require('./settings.js'); const cache = new CacheService(settings.timeToLive); +// Repopulate cache when ttl for key expires cache.cache.on('expired', (key, value) => { ContributorService.flushAll(); console.info("Cache flushed automatically on expiration for key ", key); @@ -14,7 +18,12 @@ cache.cache.on('expired', (key, value) => { }); const ContributorService = { getContributorByOrganisation(organisationName) { - return cache.get(organisationName, contributors(organisationName).then(result=>{ + /* + Data is fetched from Repository layer if not found in cache. + getContributors(organisationName) is lower level Repository method, which + can be replaced with some other data source. + */ + return cache.get(organisationName, getContributors(organisationName).then(result => { return result; })).then(contributors => { return contributors; diff --git a/src/Routes.js b/src/Routes.js index f17183a..a01165a 100644 --- a/src/Routes.js +++ b/src/Routes.js @@ -1,3 +1,6 @@ +/* + Top level REST APIs for data accessing +*/ const ContributorService = require('./ContributorService.js'); const settings = require('./settings.js'); const cors = require('cors'); @@ -7,6 +10,7 @@ const appRouter = function(app){ app.get('/contributors',cors(),async (req,res)=>{ res.status(200).send(await ContributorService.getContributorByOrganisation(settings.organisation)); }); + app.delete('/flushAll',cors(),(req,res)=>{ ContributorService.flushAll(); res.status(200).send("Successfully flushed the cache"); diff --git a/src/settings.js b/src/settings.js index 4020792..56167d4 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1,6 +1,6 @@ /* In order for this to work you MUST set a token. You probably want to set -an organisation and tags as well to suit you. +an organisation and cache settings(time to live) as well to suit you. ================================= 1. Get a token. Where do tokens come from? @@ -9,9 +9,11 @@ See here: https://github.com/blog/1509-personal-api-tokens 2. Set env vars. To set the vars on a Mac or Linux environment, run something like this in your terminal, but replace everything after the = sign with your own content. No spaces allowed. + EXPORT CONTRIBUTORS_ACCESS_TOKEN=1234356647856878 EXPORT CONTRIBUTORS_ORG=yourgithuborghere EXPORT CONTRIBUTORS_TTL=time_to_live_cache + optional - set the port this app is served on, e.g.: EXPORT PORT=3333 ================================= From 029f2e49d2866b3f80ad91ec2ed0afd15b404d33 Mon Sep 17 00:00:00 2001 From: omkar Date: Tue, 3 Dec 2019 22:44:19 +0530 Subject: [PATCH 2/3] updated readme to include startup steps --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e28962..81b6064 100644 --- a/README.md +++ b/README.md @@ -1 +1,10 @@ -# cross-repo-contributor-list \ No newline at end of file +# cross-repo-contributor-list + +Small node server to collect contributors from github for an organisation and expose an API which can be consumed by the frontend. Uses server caching to faster serve data. + +Refer settings.js for changing parameters such as Organisation, cache eviction time and Github access token. + +Steps to start server: +1] Update settings.js +2] npm init +3] npm start \ No newline at end of file From 4996c7061fb80e349c477a9b17f534a2fcf3bfe4 Mon Sep 17 00:00:00 2001 From: omkar Date: Tue, 3 Dec 2019 23:30:28 +0530 Subject: [PATCH 3/3] edited keywords --- README.md | 8 ++++---- src/ContributorRepository.js | 16 ++++++++-------- src/Routes.js | 2 +- src/settings.js | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 81b6064..63e5b68 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # cross-repo-contributor-list -Small node server to collect contributors from github for an organisation and expose an API which can be consumed by the frontend. Uses server caching to faster serve data. +Small node server to collect contributors from github for an organization and expose an API which can be consumed by the frontend. Uses server caching to faster serve data. -Refer settings.js for changing parameters such as Organisation, cache eviction time and Github access token. +Refer settings.js for changing parameters such as Organization, cache eviction time and Github access token. Steps to start server: 1] Update settings.js -2] npm init -3] npm start \ No newline at end of file +2] npm i +3] npm start diff --git a/src/ContributorRepository.js b/src/ContributorRepository.js index b270128..182a6fe 100644 --- a/src/ContributorRepository.js +++ b/src/ContributorRepository.js @@ -8,10 +8,10 @@ const Repository = require('./Repository.js'); const Profile = require('./Profile.js'); const settings = require('./settings.js'); -function getContributorsByOrg(organisation) { - return getReposByOrg(organisation).then((data) => { +function getContributorsByOrg(organization) { + return getReposByOrg(organization).then((data) => { return async.mapLimit(data, 10, async function (repository) { - const contributorPromise = await getContributorsByRepo(organisation, repository["name"]); + const contributorPromise = await getContributorsByRepo(organization, repository["name"]); return contributorPromise; }); }).then(async (result) => { @@ -30,7 +30,7 @@ function getContributorsByRepo(org, repositoryName) { headers: { Accept: "application/vnd.github.v3+json", Authorization: settings.access_token, - "User-Agent": settings.organisation + "User-Agent": settings.organization } }); // collect all contributors for a repo, map them to Profile class @@ -47,18 +47,18 @@ function getContributorsByRepo(org, repositoryName) { }); } -function getReposByOrg(organisation) { +function getReposByOrg(organization) { return new Promise((resolve, reject) => { var options = githubApi.repos.listForOrg.endpoint.merge({ - org: organisation, + org: organization, type: "all", headers: { Accept: "application/vnd.github.v3+json", Authorization: settings.access_token, - "User-Agent": settings.organisation + "User-Agent": settings.organization } }); - // collect all repos for an organisation, map the data to Repositor class and return + // collect all repos for an organization, map the data to Repositor class and return githubApi.paginate(options) .then(data => { data = data diff --git a/src/Routes.js b/src/Routes.js index a01165a..f693c43 100644 --- a/src/Routes.js +++ b/src/Routes.js @@ -8,7 +8,7 @@ const cors = require('cors'); const appRouter = function(app){ app.get('/contributors',cors(),async (req,res)=>{ - res.status(200).send(await ContributorService.getContributorByOrganisation(settings.organisation)); + res.status(200).send(await ContributorService.getContributorByOrganisation(settings.organization)); }); app.delete('/flushAll',cors(),(req,res)=>{ diff --git a/src/settings.js b/src/settings.js index 56167d4..32ca113 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1,6 +1,6 @@ /* In order for this to work you MUST set a token. You probably want to set -an organisation and cache settings(time to live) as well to suit you. +an organization and cache settings(time to live) as well to suit you. ================================= 1. Get a token. Where do tokens come from? @@ -23,7 +23,7 @@ If you prefer, you _can_ set them simply by editing this file, e.g. for const to const token = process.env.CONTRIBUTORS_ACCESS_TOKEN; const settings = { - "organisation": process.env.CONTRIBUTORS_ORG || "Intermine", + "organization": process.env.CONTRIBUTORS_ORG || "Intermine", "access_token": "token "+token, "timeToLive": process.env.CONTRIBUTORS_TTL || 1296000 //15*24*60*60 ie 15 days }