diff --git a/Dockerfile.client b/Dockerfile.client
index 279cd274..0dd169a9 100644
--- a/Dockerfile.client
+++ b/Dockerfile.client
@@ -5,6 +5,8 @@ COPY client/package.json /usr/src/app
WORKDIR /usr/src/app
RUN npm install --legacy-peer-deps
+RUN npm install d3 --legacy-peer-deps
+RUN npm install react-select --legacy-peer-deps
COPY client /usr/src/app
RUN npm run build
diff --git a/app/__init__.py b/app/__init__.py
index be1c8299..3784e7dd 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -13,6 +13,7 @@
from .api.FollowRepository import FollowRepository
from .api.Auth import Auth
from .api.ForkList import ForkList
+from .api.Progress import Progress
from .api.ForkClustering import ForkClustering
from .db import initialize_db
from .loginmanager import login_manager
@@ -105,6 +106,12 @@ def serve(path):
resource_class_kwargs={"jwt": jwt},
)
+ api.add_resource(
+ Progress,
+ "/flask/progress",
+ resource_class_kwargs={"jwt": jwt},
+ )
+
# TODO: get correct host, broker and backend depending on environment
redis_host = "redis://redis:6379/0"
celery.conf.broker_url = redis_host
diff --git a/app/analyse/analyser.py b/app/analyse/analyser.py
index 851a80c1..6ce7151e 100644
--- a/app/analyse/analyser.py
+++ b/app/analyse/analyser.py
@@ -151,6 +151,38 @@ def get_commit_number_per_hour(repo, access_token):
commit_info = res.json()
return commit_info
+def get_commit_number_per_week(repo, access_token):
+
+ request_url = "https://api.github.com/repos/%s/stats/participation" % repo
+
+ res = requests.get(
+ url=request_url,
+ headers={
+ "Accept": "application/json",
+ "Authorization": "token {}".format(access_token),
+ },
+ )
+ commit_info = res.json()
+ print(commit_info)
+ if ('all' in commit_info.keys()):
+ return commit_info['all']
+ else:
+ return [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+
+def get_commit_number_per_hour(repo, access_token):
+
+ request_url = "https://api.github.com/repos/%s/stats/commit_activity" % repo
+
+ res = requests.get(
+ url=request_url,
+ headers={
+ "Accept": "application/json",
+ "Authorization": "token {}".format(access_token),
+ },
+ )
+ commit_info = res.json()
+ return commit_info
+
@celery.task
def start_analyse(repo, access_token):
"""Start analyse on repo using github_api_caller(contains personal access token)
@@ -186,7 +218,7 @@ def start_analyse(repo, access_token):
current_app.config["LOCAL_DATA_PATH"] + "/" + repo + "/forks_list.json"
)
- active_forks = get_active_forks(repo,access_token )
+ active_forks = get_active_forks(repo, access_token)
if current_app.config["USE_LOCAL_FORKS_LIST"] and os.path.exists(forks_list_path):
with open(forks_list_path) as read_file:
diff --git a/app/analyse/project_updater.py b/app/analyse/project_updater.py
index edfee9b2..afb343c4 100644
--- a/app/analyse/project_updater.py
+++ b/app/analyse/project_updater.py
@@ -87,9 +87,9 @@ def work(self):
if os.path.exists(self.diff_result_path):
with open(self.diff_result_path) as read_file:
compare_result = json.load(read_file)
- else:
- # local file not exist
- return
+ # else:
+ # # local file not exist
+ # return
else:
# If the compare result is not crawled, start to crawl.
splitForkName = self.fork_name.split("/")
diff --git a/app/api/FollowRepository.py b/app/api/FollowRepository.py
index 560a850b..a875c8d9 100644
--- a/app/api/FollowRepository.py
+++ b/app/api/FollowRepository.py
@@ -115,4 +115,5 @@ def post(self):
"timesForked": res["forks_count"],
"repo": res["full_name"],
},
+ "analyser_progress": db_find_project(repo)["analyser_progress"]
}
diff --git a/app/api/ForkList.py b/app/api/ForkList.py
index 3a500f9e..6c766411 100644
--- a/app/api/ForkList.py
+++ b/app/api/ForkList.py
@@ -198,7 +198,7 @@ def post(self):
forks_info = ProjectFork.objects(project_name=repo)
fork = forks_info[index]
return_list = []
-
+
return_list.append(
{
"fork_name": fork["fork_name"],
diff --git a/app/api/Progress.py b/app/api/Progress.py
new file mode 100644
index 00000000..8f9a61dd
--- /dev/null
+++ b/app/api/Progress.py
@@ -0,0 +1,71 @@
+from flask_restful import Resource
+from flask_jwt_extended import get_jwt_identity
+from flask_jwt_extended import jwt_required
+import json
+from flask import request
+import requests
+from ..models import User, ProjectFork, Project
+from ..analyse.compare_changes_crawler import fetch_commit_list, fetch_diff_code
+from ..analyse.analyser import get_active_forks
+from ..analyse.analyser import get_commit_number_per_week
+from ..analyse.analyser import get_commit_number_per_hour
+from rake_nltk import Rake
+
+def db_find_project(project_name):
+ return Project.objects(project_name=project_name).first()
+
+
+class Progress(Resource):
+ def __init__(self, jwt):
+ self.jwt = jwt
+
+ @jwt_required()
+ def post(self):
+
+ current_user = get_jwt_identity()
+ _user = User.objects(username=current_user).first()
+
+ req_data = request.get_json()
+ repoName = req_data.get("repo")
+ index = req_data.get("index")
+ repo = repoName
+
+ forks_info = ProjectFork.objects(project_name=repo)
+ fork = forks_info[index]
+ return_list = []
+
+ return_list.append(
+ {
+ "fork_name": fork["fork_name"],
+ "project_name": fork["project_name"],
+ "num_changed_files": fork["total_changed_file_number"],
+ "num_changed_lines": fork["total_changed_line_number"],
+ "changed_files": fork["file_list"],
+ "key_words": fork["key_words"],
+ "tags": fork["tags"],
+ "total_commit_number": fork["total_commit_number"],
+ "last_committed_time": str(fork["last_committed_time"]),
+ "created_time": str(fork["created_time"]),
+ "weekly_commit_freq": get_commit_number_per_week(fork["fork_name"], _user.github_access_token),
+ "hourly_commit_freq": get_commit_number_per_hour(fork["fork_name"], _user.github_access_token),
+ }
+ )
+
+ return {"forks": return_list}
+
+ @jwt_required()
+ def get(self):
+
+
+ current_user = get_jwt_identity()
+ _user = User.objects(username=current_user).first()
+
+ req_data = request.args
+ repoName = req_data.get("repo")
+ repo = repoName
+
+ forks_info = ProjectFork.objects(project_name=repo)
+
+ return len(forks_info)
+
+
diff --git a/client/src/App.js b/client/src/App.js
index c2f59ebf..727af0da 100644
--- a/client/src/App.js
+++ b/client/src/App.js
@@ -22,6 +22,7 @@ import LoginModal from "./LoginModal";
import ForkCluster from "./ForkCluster";
import { getUserLogin } from "./repository";
import Forklist from "./Forklist";
+import ForkGraph from "./ForkGraph";
import DrawerCard from "./DrawerCard";
const theme = createTheme({
@@ -163,6 +164,16 @@ const App = () => {
)
}
/>
+ {
+ console.log("repo nav", repo);
+ navigate(`/visual/${repo}`, { replace: true });
+ };
+
return (
@@ -49,12 +55,11 @@ const FollowedRepositoryCard = ({
style={{ background: SECONDARY }}
expandIcon={
-
+
+
{repo}
-
-
+
+
+
+
-
diff --git a/client/src/ForkGraph.jsx b/client/src/ForkGraph.jsx
new file mode 100644
index 00000000..2d5d7b81
--- /dev/null
+++ b/client/src/ForkGraph.jsx
@@ -0,0 +1,184 @@
+
+import React, { useState, forwardRef, useEffect, useCallback } from "react";
+import Select from 'react-select';
+import { useParams } from "react-router-dom";
+import { Box, Typography, Card } from "@mui/material";
+import { getActiveForksNum } from "./repository";
+import { postProgress } from "./repository";
+import Loading from "./common/Loading"
+import ForkRank from "./ForkRank";
+
+const ForkGraph = () => {
+
+ const options = [
+ { value: 'daily', label: 'Daily commits within last month' },
+ { value: 'weekly', label: 'Weekly commits within last year' },
+ ];
+
+ const { repo1, repo2 } = useParams();
+ const [data, setData] = useState(null);
+ const [selection, setSelectionState] = useState(null);
+ const [interval, setIntervalState] = useState(null);
+ const [forkNames, setForkNames] = useState(null);
+ const [forkList, setForkList] = useState(null);
+ const [activeForksNum, setActiveForksNum] = useState(0);
+ const [progress, setProgress] = useState(0);
+ const [counter, setCounter] = useState(0);
+
+ const getDataList = (forks_list) => {
+ //EXTRACT TOP TEN
+ let datalist = []
+ for (let i = 0; i < forks_list.length; i++) {
+ let fork = forks_list[i]
+ console.log(fork)
+ let commit_list = fork["weekly_commit_freq"]
+ for (let j = 0; j < commit_list.length; j++) {
+ let ith_week_dic = {}
+ ith_week_dic["week"] = j
+ ith_week_dic["commits"] = commit_list[j]
+ ith_week_dic["fork_name"] = fork["fork_name"]
+ datalist.push(ith_week_dic)
+ }
+ }
+ return datalist
+}
+
+const getDailyCommit = (forks_list) => {
+ //EXTRACT TOP TEN
+ let datalist = []
+ for (let i = 0; i < forks_list.length; i++) {
+ let fork = forks_list[i]
+ let commit_list;
+ if (Object.keys(fork["hourly_commit_freq"]).length !== 52){
+ commit_list = [{"days": new Array(7).fill(0)},
+ {"days": new Array(7).fill(0)},
+ {"days": new Array(7).fill(0)},
+ {"days": new Array(7).fill(0)}]
+ } else {
+ commit_list = fork["hourly_commit_freq"].slice(-4)
+ }
+ let index = 0;
+ for (let j = 0; j < 4; j++) {
+ for(let k = 0 ; k < 7; k++){
+ let ith_week_dic = {}
+ ith_week_dic["week"] = index
+ ith_week_dic["commits"] = commit_list[j]["days"][k]
+ ith_week_dic["fork_name"] = fork["fork_name"]
+ index += 1
+ datalist.push(ith_week_dic)
+ }
+ }
+ }
+ return datalist
+}
+
+const updateData = (selection) => {
+ switch(selection) {
+ case 'weekly':
+ setIntervalState(Array.from(Array(52).keys()))
+ setData(getDataList(forkList))
+ break;
+ case 'daily':
+ setIntervalState(Array.from(Array(28).keys()));
+ setData(getDailyCommit(forkList));
+ break;
+ default:
+ // code block
+ }
+}
+
+const handleChange = (selectedOption) => {
+ setSelectionState(selectedOption);
+ console.log(`Option selected:`, selectedOption);
+ updateData(selectedOption['value'])
+ console.log(data)
+ console.log(interval)
+};
+
+const getForkNames = (forks_list) => {
+ let forkNames = []
+ // for (let i = 0; i < forks_list.length && i < 10; i++) {
+ for (let i = 0; i < forks_list.length; i++) {
+ forkNames.push(forks_list[i]["fork_name"])
+ }
+ return forkNames
+}
+
+const fetchForks = useCallback(async (repo) => {
+ console.log('repo1',repo1);
+ console.log('repo2', repo2);
+
+ //get total num of forks needs to be fetched
+ const active_fork_num = await getActiveForksNum(repo);
+ console.log("Active forks number is ", active_fork_num.data)
+ setActiveForksNum(active_fork_num.data)
+
+ let total_list = []
+ let counter = 0
+ while (counter < active_fork_num.data) {
+ let res = await postProgress(repo, counter);
+ console.log(res.data.forks[0])
+ total_list.push(res.data.forks[0])
+ counter += 1
+ setCounter(counter)
+ console.log(counter)
+ setProgress(counter/active_fork_num.data * 100)
+ console.log(progress)
+ }
+ console.log(total_list)
+
+ // const response2 = await getProgress(repo);
+ // console.log("Fetching nnnnnnn for ", response2)
+ // const response = await getRepoForks(repo);
+ // console.log("Fetching forks list for ", repo)
+ // setForkList(response.data.forks);
+ setForkList(total_list);
+ // let data = getDailyCommit(response.data.forks)
+ let data = getDailyCommit(total_list)
+ let interval = Array.from(Array(28).keys());
+ console.log(data)
+ setIntervalState(interval)
+ setData(data)
+ setForkNames(getForkNames(total_list))
+}, []);
+
+ useEffect(() => {
+ const repo = repo1 + "/" + repo2;
+ fetchForks(repo);
+ }, [fetchForks]);
+
+ return (
+ data && forkNames ?
+
+
+
+ Fork Activeness Visualization
+
+
+
+ Given the overwhelming numbers of forks in many popular repositories,
+ INFOX applied a bump graph to visualize each fork's activeness. The number inside each bump represents the
+ number of commits rank for this fork within a specific time interval. Currently the visualization graph supports
+ two time interval options, the weekly commits rank within last year and daily commits rank within last month.
+
+
+ The specific commit number within that day/week could be shown when hovering over the specifc bump.
+
+
+
+
+
+
+
+
{
+ const array = [];
+ matrix.forEach((d) => array.push(d[i]));
+ array.sort((a, b) => b.commits - a.commits);
+ array.forEach((d, j) => d.rank = j);
+ });
+
+ //before step 2
+ // get ranking
+ const chartData = matrix;
+ const len = interval.length - 1;
+ const ranking = chartData.map((d, i) => ({fork_name: forkNames[i], first: d[0].rank, last: d[len].rank}));
+ // get color
+ const color = d3.scaleOrdinal(d3.schemeTableau10)
+ .domain(seq(0, ranking.length))
+
+ const left = ranking.sort((a, b) => a.first - b.first).map((d) => d.fork_name);
+ const right = ranking.sort((a, b) => a.last - b.last).map((d) => d.fork_name);
+
+ const strokeWidth = d3.scaleOrdinal()
+ .domain(["default", "transit", "compact"])
+ .range([5, bumpRadius * 2 + 2, 2]);
+ const drawingStyle = 'default';
+ const bumpRadius = 8
+ const by = d3.scalePoint()
+ .domain(seq(0, ranking.length))
+ .range([margin.top, height - margin.bottom - padding])
+
+ function restore() {
+ series.transition().duration(500)
+ .attr("fill", s => color(s[0].rank)).attr("stroke", s => color(s[0].rank));
+ restoreTicks(leftY);
+ restoreTicks(rightY);
+
+ function restoreTicks(axis) {
+ axis.selectAll(".tick text")
+ .transition().duration(500)
+ .attr("font-weight", "normal").attr("fill", "black");
+ }
+ }
+
+ function highlight(e, d) {
+ this.parentNode.appendChild(this);
+ series.filter(s => s !== d)
+ .transition().duration(500)
+ .attr("fill", "#ddd").attr("stroke", "#ddd");
+ markTick(leftY, 0);
+ markTick(rightY, interval.length - 1);
+
+ function markTick(axis, pos) {
+ axis.selectAll(".tick text").filter((s, i) => i === d[pos].rank)
+ .transition().duration(500)
+ .attr("font-weight", "bold")
+ .attr("fill", color(d[0].rank));
+ }
+ }
+
+ //dashed line
+ svg.append("g")
+ .attr("transform", `translate(${margin.left + padding},0)`)
+ .selectAll("path")
+ .data(seq(0, interval.length))
+ .join("path")
+ .attr("stroke", "#ccc")
+ .attr("stroke-width", 2)
+ .attr("stroke-dasharray", "5,5")
+ .attr("d", d => d3.line()([[bx(d), 0], [bx(d), height - margin.bottom]]));
+
+ const series = svg.selectAll(".series")
+ .data(chartData)
+ .join("g")
+ .attr("class", "series")
+ .attr("opacity", 1)
+ .attr("fill", d => color(d[0].rank))
+ .attr("stroke", d => color(d[0].rank))
+ .attr("transform", `translate(${margin.left + padding},0)`)
+ .on("mouseover", highlight)
+ .on("mouseout", restore);
+
+
+
+
+ series.selectAll("path")
+ .data(d => d)
+ .join("path")
+ .attr("stroke-width", strokeWidth(drawingStyle))
+ .attr("d", (d, i) => {
+ if (d.next)
+ return d3.line()([[bx(i), by(d.rank)], [bx(i + 1), by(d.next.rank)]]);
+ })
+
+ const title = g => g.append("title")
+ .text((d, i) => `${d.fork_name} - ${interval[i]}\nRank: ${d.commits.rank + 1}\nCommits: ${d.commits.commits}`)
+
+ const bumps = series.selectAll("g")
+ .data((d, i) => d.map(v => ({fork_name: forkNames[i], commits: v, first: d[0].rank})))
+ .join("g")
+ .attr("transform", (d, i) => `translate(${bx(i)},${by(d.commits.rank)})`)
+ //.call(g => g.append("title").text((d, i) => `${d.fork_name} - ${interval[i]}\n${toCurrency(d.commits.commits)}`));
+ .call(title);
+
+ const ax = d3.scalePoint()
+ .domain(interval)
+ .range([margin.left + padding, width - margin.right - padding]);
+
+ const y = d3.scalePoint()
+ .range([margin.top, height - margin.bottom - padding]);
+
+ const compact = drawingStyle === "compact";
+ bumps.append("circle").attr("r", compact ? 5 : bumpRadius);
+ bumps.append("text")
+ .attr("dy", compact ? "-0.75em" : "0.35em")
+ .attr("fill", compact ? null : "white")
+ .attr("stroke", "none")
+ .attr("text-anchor", "middle")
+ .style("font-weight", "bold")
+ .style("font-size", "14px")
+ .text(d => d.commits.rank + 1);
+
+
+
+ const drawAxis = (g, x, y, axis, domain) => {
+ g.attr("transform", `translate(${x},${y})`)
+ .call(axis)
+ .selectAll(".tick text")
+ .attr("font-size", "11px");
+
+ if (!domain) g.select(".domain").remove();
+ }
+
+
+
+ //Axis
+ svg.append("g").call(g => drawAxis(g, 0, height - margin.top - margin.bottom + padding, d3.axisBottom(ax), true));
+ const leftY = svg.append("g").call(g => drawAxis(g, margin.left, 0, d3.axisLeft(y.domain(left))));
+ const rightY = svg.append("g").call(g => drawAxis(g, width - margin.right, 0, d3.axisRight(y.domain(right))));
+
+ },
+ [data.length]
+ );
+
+ return (
+
+
+ );
+}
+export default ForkRank;
\ No newline at end of file
diff --git a/client/src/Forklist.jsx b/client/src/Forklist.jsx
index 05376a16..086326b6 100644
--- a/client/src/Forklist.jsx
+++ b/client/src/Forklist.jsx
@@ -29,6 +29,8 @@ import MuiAlert from "@mui/material/Alert";
import Stack from "@mui/material/Stack";
import { differenceWith, intersectionWith, isEqual } from "lodash";
import { getRepoForks } from "./repository";
+import { getActiveForksNum } from "./repository";
+import { postProgress } from "./repository";
import { getTotalForksNumber } from "./repository";
import Loading from "./common/Loading"
import Filter from "./common/Filter";
@@ -358,6 +360,7 @@ const EnhancedTable = ({ data }) => {
const handleChangePage = (event, newPage) => {
setPage(newPage);
+ console.log(newPage)
};
const handleChangeRowsPerPage = (event) => {
@@ -817,7 +820,7 @@ const ForkList = () => {
const fetchForks = useCallback(async (repo) => {
console.log('repo1',repo1);
console.log('repo2', repo2);
-
+
//get total num of forks needs to be fetched
const active_fork_num = await getTotalForksNumber(repo);
console.log("Active forks number is ", active_fork_num.data)
diff --git a/client/src/ImportRepositoryCard.jsx b/client/src/ImportRepositoryCard.jsx
index 820d1a1b..37fa1cac 100644
--- a/client/src/ImportRepositoryCard.jsx
+++ b/client/src/ImportRepositoryCard.jsx
@@ -35,8 +35,12 @@ const ImportRepositoryCard = ({ name, description, language, timesForked, follow
onClick={async (event) => {
event.stopPropagation();
setIsLoading(true);
- const res = await postFollowRepository(name);
+ let res = await postFollowRepository(name);
console.log("res", res);
+ while (res.data.analyser_progress !== "100%"){
+ res = await postFollowRepository(name);
+ console.log("res", res);
+ }
onFollow(res.data);
setIsLoading(false);
}}
diff --git a/client/src/SearchGithubRow.jsx b/client/src/SearchGithubRow.jsx
index 3ce3b7de..4a8a04a7 100644
--- a/client/src/SearchGithubRow.jsx
+++ b/client/src/SearchGithubRow.jsx
@@ -20,6 +20,7 @@ const SearchGithubRow = ({
followedRepos,
}) => {
const [isLoading, setIsLoading] = useState(false);
+ const [progress, setProgress] = useState("0%");
return (
@@ -48,14 +49,18 @@ const SearchGithubRow = ({
variant="outlined"
onClick={async () => {
setIsLoading(true);
- const res = await postFollowRepository(name);
+ let res = await postFollowRepository(name);
+ while (res.data.analyser_progress !== "100%") {
+ res = await postFollowRepository(name);
+ setProgress(res.data.analyser_progress)
+ }
console.log("res", res);
onFollow(res.data);
setIsLoading(false);
}}
disabled={isLoading}
>
- {isLoading ? "Following..." : "Follow"}
+ {isLoading ? "Following..., progress is " + progress : "Follow"}
) : (
+
+
+
+ {`${Math.round(
+ value,
+ )}%`}
+
+
+ );
+}
+
+const LinearLoading = ({
+ loadingMessage,
+ progress
+}) => {
+ return (
+
+
+
+ {loadingMessage ? (
+ {loadingMessage}
+ ) : null}
+
+
+ );
+};
+
+LinearLoading.defaultProps = {
+ loadingMessage: null,
+};
+
+export default LinearLoading;
\ No newline at end of file
diff --git a/client/src/common/Loading.jsx b/client/src/common/Loading.jsx
index b3f098bb..c9b62617 100644
--- a/client/src/common/Loading.jsx
+++ b/client/src/common/Loading.jsx
@@ -12,7 +12,7 @@ const Loading = ({ loadingMessage }) => {
direction="column"
alignItems="center"
justifyContent="center"
- style={{ minHeight: "100%" }}
+ style={{ minHeight: "100%", padding: 10}}
>
diff --git a/config.py b/config.py
index bc94b38f..4ef27e08 100644
--- a/config.py
+++ b/config.py
@@ -15,7 +15,7 @@ class Config:
MONGODB_SETTINGS = {
"db": "forks-insights",
- "host": "mongodb+srv://admin:infox123@forks-insights.lktn4iv.mongodb.net/INFOX?retryWrites=true&w=majority&ssl=true", # Replace this with the host string mongodb+srv://....
+ "host": "mongodb+srv://admin:infox123@forks-insights.lktn4iv.mongodb.net/INFOX", # Replace this with the host string mongodb+srv://....
}
# TODO: get this from environment