diff --git a/.gitignore b/.gitignore
index 5f6160a..458b38f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -329,3 +329,6 @@ gate_analytics/media/
.env
.envs/*
!.envs/.local/
+
+
+data/
diff --git a/.idea/gate_analytics.iml b/.idea/gate_analytics.iml
index fe07f0d..59852f4 100644
--- a/.idea/gate_analytics.iml
+++ b/.idea/gate_analytics.iml
@@ -13,7 +13,9 @@
-
+
+
+
@@ -37,4 +39,4 @@
-
\ No newline at end of file
+
diff --git a/config/settings/base.py b/config/settings/base.py
index 0551ec6..263a8e0 100644
--- a/config/settings/base.py
+++ b/config/settings/base.py
@@ -76,6 +76,7 @@
LOCAL_APPS = [
"gate_analytics.users.apps.UsersConfig",
# Your stuff: custom apps go here
+ "gate_analytics.analytics.apps.AnalyticsConfig",
]
# https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
diff --git a/gate_analytics/analytics/__init__.py b/gate_analytics/analytics/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/gate_analytics/analytics/admin.py b/gate_analytics/analytics/admin.py
new file mode 100644
index 0000000..11e9434
--- /dev/null
+++ b/gate_analytics/analytics/admin.py
@@ -0,0 +1,21 @@
+from django.contrib import admin
+
+from .models import (
+ AnswerKey,
+ AnswerKeyItem,
+ Exam,
+ Question,
+ Submission,
+ SubmissionEvaluation,
+ SubmissionItem,
+ Topic,
+)
+
+admin.site.register(Exam)
+admin.site.register(Question)
+admin.site.register(AnswerKey)
+admin.site.register(AnswerKeyItem)
+admin.site.register(Submission)
+admin.site.register(SubmissionEvaluation)
+admin.site.register(SubmissionItem)
+admin.site.register(Topic)
diff --git a/gate_analytics/analytics/algos/__init__.py b/gate_analytics/analytics/algos/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/gate_analytics/analytics/algos/evaluate.py b/gate_analytics/analytics/algos/evaluate.py
new file mode 100644
index 0000000..4c4927a
--- /dev/null
+++ b/gate_analytics/analytics/algos/evaluate.py
@@ -0,0 +1,5 @@
+from gate_analytics.analytics.models import AnswerKey, Submission, SubmissionEvaluation
+
+
+def evaluate(submission: Submission, key: AnswerKey) -> SubmissionEvaluation:
+ raise NotImplementedError()
diff --git a/gate_analytics/analytics/algos/parser.py b/gate_analytics/analytics/algos/parser.py
new file mode 100644
index 0000000..85a2991
--- /dev/null
+++ b/gate_analytics/analytics/algos/parser.py
@@ -0,0 +1,42 @@
+from requests_html import HTML, HTMLSession
+
+from gate_analytics.analytics.models import (
+ QuestionType,
+ Submission,
+ SubmissionItemStatus,
+)
+from gate_analytics.utils.itertools import make_batch
+from gate_analytics.utils.text import deduplicate_slashes
+
+
+def parse(url: str) -> Submission:
+ url = deduplicate_slashes(url)
+ html: HTML = HTMLSession().get(url).html
+ for question_pnl in html.find(".questionPnlTbl"):
+ for img in question_pnl.find(".questionRowTbl img"):
+ src = deduplicate_slashes(img.attrs["src"])
+ print(src)
+ metas = make_batch(question_pnl.find(".menu-tbl td"), size=2)
+ assert metas[0][0].full_text == "Question Type :"
+ question_type = QuestionType.from_text(metas[0][1].full_text)
+ assert metas[1][0].full_text == "Question ID :"
+ question_uid = metas[1][1].full_text.strip()
+ assert metas[2][0].full_text == "Status :"
+ status = SubmissionItemStatus.from_text(metas[2][1].full_text)
+ answer = ""
+ if status == SubmissionItemStatus.Answered and question_type in [
+ QuestionType.MCQ,
+ QuestionType.MSQ,
+ ]:
+ assert metas[3][0].full_text == "Chosen Option :"
+ answer = metas[3][1].full_text.strip()
+ elif (
+ status == SubmissionItemStatus.Answered
+ and question_type == QuestionType.NAT
+ ):
+ prompt, value = question_pnl.find(".questionRowTbl td")[-2:]
+ assert prompt.full_text == "Given Answer :"
+ answer = value.full_text.strip()
+ print(question_type.name, question_uid, status.name, answer)
+ print("-" * 40)
+ raise NotImplementedError()
diff --git a/gate_analytics/analytics/apps.py b/gate_analytics/analytics/apps.py
new file mode 100644
index 0000000..8941194
--- /dev/null
+++ b/gate_analytics/analytics/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class AnalyticsConfig(AppConfig):
+ name = "gate_analytics.analytics"
diff --git a/gate_analytics/analytics/migrations/0001_initial.py b/gate_analytics/analytics/migrations/0001_initial.py
new file mode 100644
index 0000000..b35911b
--- /dev/null
+++ b/gate_analytics/analytics/migrations/0001_initial.py
@@ -0,0 +1,145 @@
+# Generated by Django 3.1.7 on 2021-02-23 18:05
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='AnswerKey',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=255)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Exam',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('branch', models.CharField(max_length=255)),
+ ('year', models.DateField()),
+ ('set', models.IntegerField()),
+ ('conducting_authority', models.TextField(blank=True)),
+ ('number_of_applicants', models.IntegerField(blank=True, null=True)),
+ ('applicants_present', models.IntegerField(blank=True, null=True)),
+ ('recommended_answer_key', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='recommended_for_exam', to='analytics.answerkey')),
+ ],
+ options={
+ 'unique_together': {('branch', 'year', 'set')},
+ },
+ ),
+ migrations.CreateModel(
+ name='Question',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('number', models.IntegerField()),
+ ('image', models.URLField()),
+ ('gateoverflow_link', models.URLField()),
+ ('type', models.IntegerField(choices=[(0, 'Mcq'), (1, 'Msq'), (2, 'Nat')], db_index=True)),
+ ('answer_choices', models.JSONField()),
+ ('max_marks', models.FloatField()),
+ ('exam', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.exam')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Submission',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('candidate_id', models.CharField(max_length=255)),
+ ('candidate_name', models.CharField(max_length=255)),
+ ('test_center_name', models.CharField(max_length=255)),
+ ('test_date', models.DateField()),
+ ('test_time_start', models.DateTimeField()),
+ ('test_time_end', models.DateTimeField()),
+ ('submission_url', models.URLField(db_index=True, unique=True)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Topic',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=255, unique=True)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='SubmissionItem',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('answer', models.TextField()),
+ ('question', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.question')),
+ ('submission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.submission')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='SubmissionEvaluation',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('attempted', models.IntegerField()),
+ ('positive_marks', models.FloatField()),
+ ('negative_marks', models.FloatField()),
+ ('net_marks', models.FloatField()),
+ ('key', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.answerkey')),
+ ('submission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.submission')),
+ ],
+ ),
+ migrations.AddField(
+ model_name='submission',
+ name='evaluations',
+ field=models.ManyToManyField(through='analytics.SubmissionEvaluation', to='analytics.AnswerKey'),
+ ),
+ migrations.AddField(
+ model_name='submission',
+ name='exam',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.exam'),
+ ),
+ migrations.AddField(
+ model_name='submission',
+ name='items',
+ field=models.ManyToManyField(through='analytics.SubmissionItem', to='analytics.Question'),
+ ),
+ migrations.AddField(
+ model_name='question',
+ name='topics',
+ field=models.ManyToManyField(blank=True, null=True, to='analytics.Topic'),
+ ),
+ migrations.CreateModel(
+ name='AnswerKeyItem',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('correct_answer', models.TextField()),
+ ('marks_to_all', models.BooleanField(default=False)),
+ ('excluded', models.BooleanField(default=False)),
+ ('key', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.answerkey')),
+ ('question', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.question')),
+ ],
+ ),
+ migrations.AddField(
+ model_name='answerkey',
+ name='answers',
+ field=models.ManyToManyField(through='analytics.AnswerKeyItem', to='analytics.Question'),
+ ),
+ migrations.AddField(
+ model_name='answerkey',
+ name='exam',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.exam'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='submission',
+ unique_together={('exam', 'candidate_id')},
+ ),
+ migrations.AlterUniqueTogether(
+ name='question',
+ unique_together={('exam', 'number')},
+ ),
+ migrations.AlterUniqueTogether(
+ name='answerkey',
+ unique_together={('exam', 'name')},
+ ),
+ ]
diff --git a/gate_analytics/analytics/migrations/__init__.py b/gate_analytics/analytics/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/gate_analytics/analytics/models.py b/gate_analytics/analytics/models.py
new file mode 100644
index 0000000..df1d4c1
--- /dev/null
+++ b/gate_analytics/analytics/models.py
@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+from django.db import models
+
+
+class Exam(models.Model):
+ branch = models.CharField(max_length=2)
+ year = models.DateField()
+ set = models.IntegerField()
+ branch_full_name = models.CharField(max_length=255)
+
+ conducting_authority = models.TextField(blank=True)
+ number_of_applicants = models.IntegerField(blank=True, null=True)
+ applicants_present = models.IntegerField(blank=True, null=True)
+
+ recommended_answer_key = models.ForeignKey(
+ "AnswerKey",
+ related_name="recommended_for_exam",
+ on_delete=models.DO_NOTHING,
+ blank=True,
+ null=True,
+ )
+
+ class Meta:
+ unique_together = (("branch", "year", "set"),)
+
+ def __str__(self):
+ return f"{self.branch}-{self.year.year}-{self.set}"
+
+
+class Topic(models.Model):
+ name = models.CharField(max_length=255, unique=True)
+
+ def __str__(self):
+ return f"{self.name}"
+
+
+class QuestionType(models.IntegerChoices):
+ MCQ = 0
+ MSQ = 1
+ NAT = 2
+
+ @staticmethod
+ def from_text(text: str) -> QuestionType:
+ return QuestionType[text.strip()[:3]]
+
+
+class Question(models.Model):
+ exam = models.ForeignKey(Exam, on_delete=models.CASCADE)
+ uid = models.CharField(max_length=255)
+
+ image = models.URLField()
+ gateoverflow_link = models.URLField()
+ type = models.IntegerField(choices=QuestionType.choices, db_index=True)
+ answer_choices = models.JSONField()
+ max_marks = models.FloatField()
+ topics = models.ManyToManyField(Topic, blank=True, null=True)
+
+ class Meta:
+ unique_together = (("exam", "uid"),)
+
+ def __str__(self):
+ return f"{self.exam}-{self.uid}"
+
+
+class AnswerKey(models.Model):
+ exam = models.ForeignKey(Exam, on_delete=models.CASCADE)
+ name = models.CharField(max_length=255)
+
+ answers = models.ManyToManyField(
+ Question, through="AnswerKeyItem", through_fields=("key", "question")
+ )
+
+ def evaluate_all_submissions(self) -> None:
+ raise NotImplementedError()
+
+ class Meta:
+ unique_together = (("exam", "name"),)
+
+ def __str__(self):
+ return f"{self.exam}-{self.name}"
+
+
+class AnswerKeyItem(models.Model):
+ key = models.ForeignKey(AnswerKey, on_delete=models.CASCADE)
+ question = models.ForeignKey(Question, on_delete=models.CASCADE)
+
+ correct_answer = models.TextField()
+ marks_to_all = models.BooleanField(default=False)
+ excluded = models.BooleanField(default=False)
+
+ def __str__(self):
+ return f"{self.key}-{self.question}"
+
+
+class Submission(models.Model):
+ exam = models.ForeignKey(Exam, on_delete=models.CASCADE)
+ candidate_id = models.CharField(max_length=255)
+ candidate_name = models.CharField(max_length=255)
+ test_center_name = models.CharField(max_length=255)
+ test_date = models.DateField()
+ test_time_start = models.DateTimeField()
+ test_time_end = models.DateTimeField()
+ submission_url = models.URLField(unique=True, db_index=True)
+
+ items = models.ManyToManyField(
+ Question, through="SubmissionItem", through_fields=("submission", "question")
+ )
+ evaluations = models.ManyToManyField(
+ AnswerKey, through="SubmissionEvaluation", through_fields=("submission", "key")
+ )
+
+ def reevaluate(self) -> None:
+ raise NotImplementedError()
+
+ class Meta:
+ unique_together = (("exam", "candidate_id"),)
+
+ def __str__(self):
+ return f"{self.exam}-{self.candidate_id}"
+
+
+class SubmissionEvaluation(models.Model):
+ submission = models.ForeignKey(Submission, on_delete=models.CASCADE)
+ key = models.ForeignKey(AnswerKey, on_delete=models.CASCADE)
+ attempted = models.IntegerField()
+ positive_marks = models.FloatField()
+ negative_marks = models.FloatField()
+ net_marks = models.FloatField()
+
+ def __str__(self):
+ return f"{self.submission}-{self.key}-{self.net_marks:.2f}"
+
+
+class SubmissionItemStatus(models.IntegerChoices):
+ Answered = 0
+ NotAttempted = 1
+
+ @staticmethod
+ def from_text(text: str) -> SubmissionItemStatus:
+ if "Not Attempted" in text:
+ return SubmissionItemStatus.NotAttempted
+ elif "Answered" in text:
+ return SubmissionItemStatus.Answered
+ else:
+ raise ValueError(text)
+
+
+class SubmissionItem(models.Model):
+ submission = models.ForeignKey(Submission, on_delete=models.CASCADE)
+ question = models.ForeignKey(Question, on_delete=models.CASCADE)
+ status = models.IntegerField(choices=SubmissionItemStatus.choices)
+ answer = models.CharField(max_length=255, blank=True, null=True)
+
+ def __str__(self):
+ return f"{self.submission}-{self.question}-{self.status}-{self.answer}"
diff --git a/gate_analytics/analytics/tests.py b/gate_analytics/analytics/tests.py
new file mode 100644
index 0000000..e69de29
diff --git a/gate_analytics/analytics/views.py b/gate_analytics/analytics/views.py
new file mode 100644
index 0000000..e69de29
diff --git a/gate_analytics/contrib/sites/migrations/0004_auto_20210223_2324.py b/gate_analytics/contrib/sites/migrations/0004_auto_20210223_2324.py
new file mode 100644
index 0000000..bf18555
--- /dev/null
+++ b/gate_analytics/contrib/sites/migrations/0004_auto_20210223_2324.py
@@ -0,0 +1,17 @@
+# Generated by Django 3.1.7 on 2021-02-23 17:54
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('sites', '0003_set_site_domain_and_name'),
+ ]
+
+ operations = [
+ migrations.AlterModelOptions(
+ name='site',
+ options={'ordering': ['domain'], 'verbose_name': 'site', 'verbose_name_plural': 'sites'},
+ ),
+ ]
diff --git a/gate_analytics/utils/itertools.py b/gate_analytics/utils/itertools.py
new file mode 100644
index 0000000..39dd13e
--- /dev/null
+++ b/gate_analytics/utils/itertools.py
@@ -0,0 +1,7 @@
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def make_batch(items: list[T], size: int) -> list[tuple[T, ...]]:
+ return list(zip(*([iter(items)] * size)))
diff --git a/gate_analytics/utils/text.py b/gate_analytics/utils/text.py
new file mode 100644
index 0000000..bed391d
--- /dev/null
+++ b/gate_analytics/utils/text.py
@@ -0,0 +1,5 @@
+def deduplicate_slashes(url: str) -> str:
+ while "//" in url:
+ url = url.replace("//", "/")
+ url = url.replace(":/", "://")
+ return url
diff --git a/requirements/base.txt b/requirements/base.txt
index 49f00cb..f9b7b15 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -18,3 +18,7 @@ django-redis==4.12.1 # https://github.com/jazzband/django-redis
# Django REST Framework
djangorestframework==3.12.2 # https://github.com/encode/django-rest-framework
django-cors-headers==3.7.0 # https://github.com/adamchainz/django-cors-headers
+
+# Custom
+# ------------------------------------------------------------------------------
+requests-html