From a4e18ef179b4adc2f72a4bce7f0783d5d7ece7df Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Fri, 31 Jul 2026 13:19:07 +0530 Subject: [PATCH 01/30] feat(viewer): viewer service module + LP auto-enrolment tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Viewer module (actors + thin controllers) wired into the lern monolith: - View lifecycle (view/start|update|end|read), assessment submit/read, summary, and the recursive nested rollup in ViewerAggregatorActor. - LP auto-enrolment: structural detection via trackablenodes, per-learner optionality, level-gated progressive enrol, cert + durable user_skills at completion. Adapters make legacy content/state + activity/agg thin translators to the viewer, with deployment_mode-driven transport (monolith: in-JVM ask/tell; distributed: HTTP). Correctness fixes in this pass: - writeAllNodeEnrolments matches rows on collectionid AND this LP's contextid, so independent same-course enrolments are no longer clobbered (design §4). - Level completion derived from persisted course-enrolment status (recompute-safe). - Optionality computed once via an in-JVM memo (no DB column). - creditSkills writes only when there are new skills. - aggregate() failures are caught and logged (fire-and-forget safety). Config: viewer_enabled, deployment_mode, viewer_service_base_url, viewer_ask_timeout_ms, hierarchy_relations_cache_ttl. Migrations: viewer.cql (prod in-place) + viewer-test-keyspace.cql (fresh keyspace for testing). --- .../resources/externalresource.properties | 6 +- .../viewer/ViewAggregateController.java | 52 + .../controllers/viewer/ViewController.java | 59 + .../viewer/ViewSummaryController.java | 73 ++ .../modules/LernServiceActorStartModule.java | 25 + modules/lern/service/conf/application.conf | 19 + modules/lern/service/conf/routes | 13 + modules/lern/service/pom.xml | 9 + .../actor/ActivityAggregatorActor.scala | 62 +- .../util/HierarchyRelationsUtil.scala | 37 +- .../HierarchyRelationsUtilCacheTest.scala | 56 + .../HierarchyRelationsUtilTrackableTest.scala | 40 + .../assessment/service/CassandraService.scala | 16 +- .../CourseBatchManagementActor.java | 82 +- .../CourseBatchManagementActorTest.java | 88 ++ .../enrolments/AssessmentAuditRecorder.scala | 9 +- .../enrolments/ContentConsumptionActor.scala | 194 +++- .../enrolments/CourseEnrolmentActor.scala | 85 ++ modules/viewer/actors/pom.xml | 175 +++ .../viewer/actor/ViewConsumptionActor.scala | 266 +++++ .../viewer/actor/ViewerAggregatorActor.scala | 378 ++++++ .../viewer/actor/ViewerRequestKeys.scala | 31 + .../viewer/actor/ViewerSummaryActor.scala | 141 +++ .../viewer/util/ProgressionPolicy.scala | 90 ++ .../actor/ViewConsumptionActorTest.scala | 145 +++ .../actor/ViewerAggregatorActorTest.scala | 70 ++ .../viewer/util/ProgressionPolicySpec.scala | 93 ++ .../migrations/viewer-test-keyspace.cql | 154 +++ modules/viewer/migrations/viewer.cql | 78 ++ modules/viewer/pom.xml | 37 + .../app/controllers/BaseController.java | 845 ++++++++++++++ .../viewer/ViewAggregateController.java | 55 + .../controllers/viewer/ViewController.java | 75 ++ .../viewer/ViewSummaryController.java | 75 ++ .../service/app/filters/AccessLogFilter.java | 86 ++ .../service/app/filters/CustomGzipFilter.java | 67 ++ .../service/app/filters/LoggingFilter.java | 42 + .../service/app/filters/ResponseFilter.scala | 40 + .../service/app/mapper/RequestMapper.java | 225 ++++ .../service/app/modules/ActorStartModule.java | 42 + .../service/app/modules/ApplicationStart.java | 83 ++ .../service/app/modules/ErrorHandler.java | 61 + .../service/app/modules/OnRequestHandler.java | 283 +++++ .../service/app/modules/StartModule.java | 29 + .../viewer/service/app/util/ACTOR_NAMES.java | 27 + modules/viewer/service/app/util/Attrs.java | 22 + .../app/util/AuthenticationHelper.java | 81 ++ modules/viewer/service/app/util/Common.java | 16 + .../service/app/util/RequestInterceptor.java | 145 +++ .../service/app/util/RequestValidator.java | 1020 +++++++++++++++++ modules/viewer/service/conf/application.conf | 404 +++++++ modules/viewer/service/conf/logback-test.xml | 1 + modules/viewer/service/conf/logback.xml | 80 ++ modules/viewer/service/conf/routes | 25 + modules/viewer/service/pom.xml | 654 +++++++++++ pom.xml | 10 + 56 files changed, 7061 insertions(+), 15 deletions(-) create mode 100644 modules/lern/service/app/controllers/viewer/ViewAggregateController.java create mode 100644 modules/lern/service/app/controllers/viewer/ViewController.java create mode 100644 modules/lern/service/app/controllers/viewer/ViewSummaryController.java create mode 100644 modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilCacheTest.scala create mode 100644 modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTrackableTest.scala create mode 100644 modules/viewer/actors/pom.xml create mode 100644 modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala create mode 100644 modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala create mode 100644 modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala create mode 100644 modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala create mode 100644 modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala create mode 100644 modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala create mode 100644 modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala create mode 100644 modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala create mode 100644 modules/viewer/migrations/viewer-test-keyspace.cql create mode 100644 modules/viewer/migrations/viewer.cql create mode 100644 modules/viewer/pom.xml create mode 100644 modules/viewer/service/app/controllers/BaseController.java create mode 100644 modules/viewer/service/app/controllers/viewer/ViewAggregateController.java create mode 100644 modules/viewer/service/app/controllers/viewer/ViewController.java create mode 100644 modules/viewer/service/app/controllers/viewer/ViewSummaryController.java create mode 100644 modules/viewer/service/app/filters/AccessLogFilter.java create mode 100644 modules/viewer/service/app/filters/CustomGzipFilter.java create mode 100644 modules/viewer/service/app/filters/LoggingFilter.java create mode 100644 modules/viewer/service/app/filters/ResponseFilter.scala create mode 100644 modules/viewer/service/app/mapper/RequestMapper.java create mode 100644 modules/viewer/service/app/modules/ActorStartModule.java create mode 100644 modules/viewer/service/app/modules/ApplicationStart.java create mode 100644 modules/viewer/service/app/modules/ErrorHandler.java create mode 100644 modules/viewer/service/app/modules/OnRequestHandler.java create mode 100644 modules/viewer/service/app/modules/StartModule.java create mode 100644 modules/viewer/service/app/util/ACTOR_NAMES.java create mode 100644 modules/viewer/service/app/util/Attrs.java create mode 100644 modules/viewer/service/app/util/AuthenticationHelper.java create mode 100644 modules/viewer/service/app/util/Common.java create mode 100644 modules/viewer/service/app/util/RequestInterceptor.java create mode 100644 modules/viewer/service/app/util/RequestValidator.java create mode 100644 modules/viewer/service/conf/application.conf create mode 100644 modules/viewer/service/conf/logback-test.xml create mode 100644 modules/viewer/service/conf/logback.xml create mode 100644 modules/viewer/service/conf/routes create mode 100644 modules/viewer/service/pom.xml diff --git a/core/sunbird-platform-common/src/main/resources/externalresource.properties b/core/sunbird-platform-common/src/main/resources/externalresource.properties index ae0f95b0..d4d16ee5 100644 --- a/core/sunbird-platform-common/src/main/resources/externalresource.properties +++ b/core/sunbird-platform-common/src/main/resources/externalresource.properties @@ -180,4 +180,8 @@ bulk_upload_org_data_size=300 sunbird_framework_read_api=/v1/framework/read sunbird.channel.create.api.url=/channel/v3/create sunbird.channel.update.api.url=/channel/v3/update -frameworkvalidation=false \ No newline at end of file +frameworkvalidation=false +viewer_service_base_url=http://lern-service:9000 +viewer_enabled=false +# LP transport topology: monolith (in-JVM) | distributed (HTTP). +deployment_mode=monolith \ No newline at end of file diff --git a/modules/lern/service/app/controllers/viewer/ViewAggregateController.java b/modules/lern/service/app/controllers/viewer/ViewAggregateController.java new file mode 100644 index 00000000..a52762e7 --- /dev/null +++ b/modules/lern/service/app/controllers/viewer/ViewAggregateController.java @@ -0,0 +1,52 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.keys.JsonKey; +import org.sunbird.request.Request; +import org.sunbird.response.ResponseCode; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Viewer resync API (monolith wiring) — recomputes a learner's collection roll-up from + * user_content_consumption. Idempotent (recompute-from-source): backfill, drift repair, or a + * recompute after a collection is republished. Mirrors the legacy POST /v1/activity/agg, but targets + * the viewer aggregator (viewer-aggregator-actor, op "aggregate"). Runs on the userId-hashed pool, so + * a resync serialises with any live /v1/view/end for that learner — no race with real-time roll-ups. + * POST /v1/view/agg { request: { userId, collectionId|courseId, contextId|batchId } } + */ +public class ViewAggregateController extends BaseController { + + @Inject + @Named("viewer-aggregator-actor") + private ActorRef viewerAggregatorActor; + + public CompletionStage agg(Http.Request httpRequest) { + try { + Request request = createAndInitRequest("aggregate", httpRequest.body().asJson(), httpRequest); + validate(request); + return actorResponseHandler(viewerAggregatorActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + private void validate(Request request) { + String userId = (String) request.get(JsonKey.USER_ID); + Object collectionId = request.get("collectionId") != null ? request.get("collectionId") : request.get(JsonKey.COURSE_ID); + if (userId == null || userId.trim().isEmpty() + || collectionId == null || collectionId.toString().trim().isEmpty()) { + throw new ProjectCommonException( + ResponseCode.mandatoryParamsMissing.getErrorCode(), + "userId and collectionId (or courseId) are mandatory", + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } +} diff --git a/modules/lern/service/app/controllers/viewer/ViewController.java b/modules/lern/service/app/controllers/viewer/ViewController.java new file mode 100644 index 00000000..f85a2917 --- /dev/null +++ b/modules/lern/service/app/controllers/viewer/ViewController.java @@ -0,0 +1,59 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.request.Request; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Viewer Service — granular view lifecycle APIs (monolith wiring). + * Dispatches to view-consumption-actor (bound in LernServiceActorStartModule). + * POST /v1/view/start -> viewStart + * POST /v1/view/update -> viewUpdate + * POST /v1/view/end -> viewEnd (synchronous recursive roll-up before responding) + */ +public class ViewController extends BaseController { + + @Inject + @Named("view-consumption-actor") + private ActorRef viewConsumptionActor; + + public CompletionStage viewStart(Http.Request httpRequest) { + return dispatch("viewStart", httpRequest); + } + + public CompletionStage viewUpdate(Http.Request httpRequest) { + return dispatch("viewUpdate", httpRequest); + } + + public CompletionStage viewEnd(Http.Request httpRequest) { + return dispatch("viewEnd", httpRequest); + } + + public CompletionStage viewRead(Http.Request httpRequest) { + return dispatch("viewRead", httpRequest); + } + + public CompletionStage assessmentSubmit(Http.Request httpRequest) { + return dispatch("viewAssess", httpRequest); + } + + public CompletionStage assessmentRead(Http.Request httpRequest) { + return dispatch("assessmentRead", httpRequest); + } + + private CompletionStage dispatch(String operation, Http.Request httpRequest) { + try { + Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + return actorResponseHandler(viewConsumptionActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } +} diff --git a/modules/lern/service/app/controllers/viewer/ViewSummaryController.java b/modules/lern/service/app/controllers/viewer/ViewSummaryController.java new file mode 100644 index 00000000..a6deeda1 --- /dev/null +++ b/modules/lern/service/app/controllers/viewer/ViewSummaryController.java @@ -0,0 +1,73 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.request.Request; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Viewer Service — read + summary APIs (monolith wiring). + * Dispatches to viewer-summary-actor (bound in LernServiceActorStartModule). + * POST /v1/summary/read + * GET /v1/summary/list/:userId + * DELETE /v1/summary/delete/:userId + */ +public class ViewSummaryController extends BaseController { + + @Inject + @Named("viewer-summary-actor") + private ActorRef viewerSummaryActor; + + public CompletionStage summaryRead(Http.Request httpRequest) { + return dispatchBody("summaryRead", httpRequest); + } + + public CompletionStage summaryList(String userId, Http.Request httpRequest) { + try { + Request request = createAndInitRequest("summaryList", httpRequest); + request.getRequest().put("userId", userId); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + public CompletionStage summaryDownload(String userId, Http.Request httpRequest) { + try { + Request request = createAndInitRequest("summaryDownload", httpRequest); + request.getRequest().put("userId", userId); + String[] fmt = httpRequest.queryString().getOrDefault("format", new String[]{"json"}); + request.getRequest().put("format", fmt.length > 0 ? fmt[0] : "json"); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + public CompletionStage summaryDelete(String userId, Http.Request httpRequest) { + try { + Request request = httpRequest.body().asJson() != null + ? createAndInitRequest("summaryDelete", httpRequest.body().asJson(), httpRequest) + : createAndInitRequest("summaryDelete", httpRequest); + request.getRequest().put("userId", userId); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + private CompletionStage dispatchBody(String operation, Http.Request httpRequest) { + try { + Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } +} diff --git a/modules/lern/service/app/modules/LernServiceActorStartModule.java b/modules/lern/service/app/modules/LernServiceActorStartModule.java index e8db0f67..8fe32c90 100644 --- a/modules/lern/service/app/modules/LernServiceActorStartModule.java +++ b/modules/lern/service/app/modules/LernServiceActorStartModule.java @@ -1,5 +1,7 @@ package modules; +import org.apache.pekko.routing.ConsistentHashingPool; +import org.apache.pekko.routing.ConsistentHashingRouter.ConsistentHashMapper; import org.apache.pekko.routing.FromConfig; import org.apache.pekko.routing.RouterConfig; import com.google.inject.AbstractModule; @@ -15,10 +17,23 @@ import org.sunbird.notification.actor.ReadNotificationActor; import org.sunbird.notification.actor.UpdateNotificationActor; import org.sunbird.observability.actor.ObservabilityReportActor; +import org.sunbird.request.Request; +import org.sunbird.viewer.actor.ViewConsumptionActor; +import org.sunbird.viewer.actor.ViewerAggregatorActor; +import org.sunbird.viewer.actor.ViewerSummaryActor; public class LernServiceActorStartModule extends AbstractModule implements PekkoGuiceSupport { private static LoggerUtil logger = new LoggerUtil(LernServiceActorStartModule.class); + // hash key for the viewer aggregator pool = userId (per-user serialization) + private static final ConsistentHashMapper viewerUserIdHashMapper = message -> { + if (message instanceof Request) { + Object uid = ((Request) message).get("userId"); + return uid != null ? uid : ""; + } + return ""; + }; + @Override protected void configure() { logger.info("LernServiceActorStartModule: Binding actors for ALL services"); @@ -53,6 +68,16 @@ protected void configure() { bindActor(ObservabilityReportActor.class, "observability-report-actor", props -> props.withRouter(config)); logger.info("Observability actors bound"); + // 5. Bind Viewer Actors (served by this monolith's /v1/view/* routes) + bindActor(ViewConsumptionActor.class, "view-consumption-actor", props -> props.withRouter(config)); + bindActor(ViewerSummaryActor.class, "viewer-summary-actor", props -> props.withRouter(config)); + // aggregator: consistent-hashing keyed on userId serialises one user's roll-ups (config router + // can't supply a hash key for plain Request messages -> would deadLetter, hence programmatic). + bindActor(ViewerAggregatorActor.class, "viewer-aggregator-actor", + props -> props.withDispatcher("pekko.actor.viewer-dispatcher") + .withRouter(new ConsistentHashingPool(8).withHashMapper(viewerUserIdHashMapper))); + logger.info("Viewer actors bound"); + logger.info("LernServiceActorStartModule: All actors bound successfully"); } } diff --git a/modules/lern/service/conf/application.conf b/modules/lern/service/conf/application.conf index bab4b64e..3304bb3b 100644 --- a/modules/lern/service/conf/application.conf +++ b/modules/lern/service/conf/application.conf @@ -131,6 +131,18 @@ pekko { } throughput = 1 } + # Viewer actors get their OWN pool: the content-consumption adapter blocks on an in-JVM ask into the + # viewer actors (monolith) — a dedicated dispatcher avoids starving the shared tracking-dispatcher. + viewer-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + throughput = 1 + } deployment { # --- USERORG ACTORS --- @@ -272,6 +284,13 @@ pekko { # --- OBSERVABILITY ACTORS --- "/observability-report-actor" { router = smallest-mailbox-pool, nr-of-instances = 4, dispatcher = rr-dispatcher } + # --- VIEWER ACTORS --- (dedicated viewer-dispatcher; see dispatcher comment above) + # viewer-aggregator-actor is NOT here: it is bound programmatically in LernServiceActorStartModule + # with a ConsistentHashingPool + hashMapper keyed on userId (config FromConfig cannot supply a + # hash key, and unhashable messages would go to deadLetters). + "/view-consumption-actor" { router = smallest-mailbox-pool, nr-of-instances = 4, dispatcher = viewer-dispatcher } + "/viewer-summary-actor" { router = smallest-mailbox-pool, nr-of-instances = 4, dispatcher = rr-dispatcher } + # --- NOTIFICATION ACTORS --- "/HealthActor" { router = smallest-mailbox-pool, nr-of-instances = 5, dispatcher = notification-dispatcher } "/HealthActor/*" { dispatcher = pekko.actor.notification-dispatcher } diff --git a/modules/lern/service/conf/routes b/modules/lern/service/conf/routes index 415877cd..06d7f9bc 100644 --- a/modules/lern/service/conf/routes +++ b/modules/lern/service/conf/routes @@ -284,3 +284,16 @@ PATCH /v1/notification/template/action/update @controllers.notification.No GET /v1/notification/template/:action @controllers.notification.NotificationTemplateController.getAction(action: String, request: play.mvc.Http.Request) + +# --- Viewer Service APIs --- +POST /v1/view/start @controllers.viewer.ViewController.viewStart(request: play.mvc.Http.Request) +POST /v1/view/update @controllers.viewer.ViewController.viewUpdate(request: play.mvc.Http.Request) +POST /v1/view/end @controllers.viewer.ViewController.viewEnd(request: play.mvc.Http.Request) +POST /v1/view/read @controllers.viewer.ViewController.viewRead(request: play.mvc.Http.Request) +POST /v1/view/agg @controllers.viewer.ViewAggregateController.agg(request: play.mvc.Http.Request) +POST /v1/assessment/submit @controllers.viewer.ViewController.assessmentSubmit(request: play.mvc.Http.Request) +POST /v1/assessment/read @controllers.viewer.ViewController.assessmentRead(request: play.mvc.Http.Request) +POST /v1/summary/read @controllers.viewer.ViewSummaryController.summaryRead(request: play.mvc.Http.Request) +GET /v1/summary/list/:userId @controllers.viewer.ViewSummaryController.summaryList(userId: String, request: play.mvc.Http.Request) +GET /v1/summary/download/:userId @controllers.viewer.ViewSummaryController.summaryDownload(userId: String, request: play.mvc.Http.Request) +DELETE /v1/summary/delete/:userId @controllers.viewer.ViewSummaryController.summaryDelete(userId: String, request: play.mvc.Http.Request) diff --git a/modules/lern/service/pom.xml b/modules/lern/service/pom.xml index 81faf8d1..adb7aac5 100644 --- a/modules/lern/service/pom.xml +++ b/modules/lern/service/pom.xml @@ -300,6 +300,15 @@ 1.0-SNAPSHOT + + + org.sunbird + viewer-actors + 1.0-SNAPSHOT + + org.sunbird diff --git a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala index b52c11f3..534f4d5d 100644 --- a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala +++ b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala @@ -14,6 +14,7 @@ import org.sunbird.request.{Request, RequestContext} import org.sunbird.response.ResponseCode import org.sunbird.enrolments.BaseEnrolmentActor import org.sunbird.helper.ServiceFactory +import org.sunbird.http.HttpClientUtil import org.sunbird.kafka.KafkaClient import org.sunbird.learner.util.Util @@ -55,7 +56,14 @@ class ActivityAggregatorActor extends BaseEnrolmentActor { val contents = if (contentsRaw != null) contentsRaw.asInstanceOf[util.List[util.Map[String, AnyRef]]] else null try { - processActivityAggregates(userId, batchId, courseId, contents, requestContext) + // Backward compatibility: the frontend keeps calling /v1/activity/agg. When the viewer is + // enabled, the Viewer Service owns the processing (see delegateToViewer); otherwise legacy. + if (isViewerEnabled) { + val token = Option(request.getContext.get(JsonKey.X_AUTH_TOKEN)).map(_.asInstanceOf[String]).orNull + delegateToViewer(userId, courseId, batchId, contents, token, requestContext) + } else { + processActivityAggregates(userId, batchId, courseId, contents, requestContext) + } sender().tell(successResponse(), self) } catch { case ex: Exception => @@ -64,6 +72,58 @@ class ActivityAggregatorActor extends BaseEnrolmentActor { } } + private def isViewerEnabled: Boolean = + java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) + + private def viewerBaseUrl: String = + Option(ProjectUtil.getConfigValue("viewer_service_base_url")).filter(StringUtils.isNotBlank) + .getOrElse("http://viewer-service:9000") + + private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) + + /** + * Backward-compat adapter for /v1/activity/agg: dispatch to the viewer view/agg APIs. Contents (if any) + * -> viewStart/viewEnd (ucc write + rollup); none -> aggregate (recompute from ucc). courseId/batchId map + * to collectionId/contextId. Transport per deployment_mode: monolith -> in-JVM tell; distributed -> HTTP. + */ + private def delegateToViewer(userId: String, courseId: String, batchId: String, + contents: util.List[util.Map[String, AnyRef]], token: String, + ctx: RequestContext): Unit = { + val headers = new util.HashMap[String, String]() {{ + put("Content-Type", "application/json") + Option(token).filter(StringUtils.isNotBlank).foreach(t => put("x-authenticated-user-token", t)) + }} + // monolith: fire-and-forget to the in-JVM viewer actor (its own rollup stays async); distributed: HTTP. + def dispatch(actorName: String, api: String, operation: String, body: util.Map[String, AnyRef]): Unit = { + if (isMonolith) { + val req = new Request(); req.setRequestContext(ctx); req.setOperation(operation); req.setRequest(body) + context.actorSelection("/user/" + actorName).tell(req, org.apache.pekko.actor.ActorRef.noSender) + } else { + val envelope = new util.HashMap[String, AnyRef]() {{ put(JsonKey.REQUEST, body) }} + HttpClientUtil.post(viewerBaseUrl + api, gson.toJson(envelope), headers, ctx) + } + } + if (contents != null && !contents.isEmpty) { + contents.asScala.foreach { c => + val status = Option(c.get(JsonKey.STATUS)).map(_.asInstanceOf[Number].intValue()).getOrElse(0) + val (op, api) = if (status >= 2) ("viewEnd", "/v1/view/end") else ("viewStart", "/v1/view/start") + dispatch("view-consumption-actor", api, op, new util.HashMap[String, AnyRef]() {{ + put("contentId", c.get(JsonKey.CONTENT_ID)) + put("collectionId", courseId) + put("contextId", batchId) + put(JsonKey.USER_ID, userId) + Option(c.get("progressdetails")).orElse(Option(c.get("progressDetails"))).foreach(pd => put("progressDetails", pd)) + }}) + } + } else { + dispatch("viewer-aggregator-actor", "/v1/view/agg", "aggregate", new util.HashMap[String, AnyRef]() {{ + put("collectionId", courseId) + put("contextId", batchId) + put(JsonKey.USER_ID, userId) + }}) + } + } + private def processActivityAggregates( userId: String, batchId: String, diff --git a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/HierarchyRelationsUtil.scala b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/HierarchyRelationsUtil.scala index 373a82ee..117c68b1 100644 --- a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/HierarchyRelationsUtil.scala +++ b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/HierarchyRelationsUtil.scala @@ -16,7 +16,10 @@ class HierarchyRelationsUtil(cassandraOperation: CassandraOperation) { private val keyspace = Option(ProjectUtil.getConfigValue("hierarchy_store_keyspace")).getOrElse("dev_hierarchy_store") private val tableName = Option(ProjectUtil.getConfigValue("hierarchy_relations_table")).getOrElse("hierarchy_relations") - def readFromDB(key: String, requestContext: RequestContext): List[String] = { + def readFromDB(key: String, requestContext: RequestContext): List[String] = + HierarchyRelationsUtil.cached(key)(readFromDBUncached(key, requestContext)) + + private def readFromDBUncached(key: String, requestContext: RequestContext): List[String] = { logger.info(requestContext, s"HierarchyRelationsUtil.readFromDB: key: $key, keyspace: $keyspace, table: $tableName") try { val queryMap = new util.HashMap[String, AnyRef]() @@ -72,6 +75,16 @@ class HierarchyRelationsUtil(cassandraOperation: CassandraOperation) { ancestors } + /** + * Ordered trackable course ids for an LP root, from the `::trackablenodes` relation + * (emitted at publish). Order is preserved (unlock order); NOT de-duplicated. Empty if absent. + */ + def getTrackableNodes(rootId: String, requestContext: RequestContext): List[String] = { + val key = s"$rootId:$rootId:trackablenodes" + logger.info(requestContext, s"HierarchyRelationsUtil: Getting trackable nodes for rootId: $rootId") + readFromDB(key, requestContext) + } + def getRequiredLeafNodes(courseId: String, collectionId: String, requestContext: RequestContext): List[String] = { logger.info(requestContext, s"HierarchyRelationsUtil: Getting required leaf nodes (excluding optional) for courseId: $courseId, collectionId: $collectionId") val leafNodes = getLeafNodes(courseId, collectionId, requestContext) @@ -84,4 +97,26 @@ class HierarchyRelationsUtil(cassandraOperation: CassandraOperation) { object HierarchyRelationsUtil { def apply(cassandraOperation: CassandraOperation): HierarchyRelationsUtil = new HierarchyRelationsUtil(cassandraOperation) + + // JVM-wide TTL cache of relationship_key -> node_ids. The hierarchy tree changes only on collection + // republish; we accept up to TTL of staleness (structure-only, self-heals on next recompute) instead + // of detecting republish. Empty results are NOT cached, so a freshly-published collection isn't held + // stale. Set hierarchy_relations_cache_ttl=0 to disable. + // ponytail: TTL eviction; go version-keyed/event-driven only if republish-during-consumption bites. + private val ttlMillis: Long = + Option(ProjectUtil.getConfigValue("hierarchy_relations_cache_ttl")) + .filter(_.trim.nonEmpty).map(_.trim.toLong).getOrElse(300L) * 1000L + private val cache = new java.util.concurrent.ConcurrentHashMap[String, (Long, List[String])]() + + private def cached(key: String)(load: => List[String]): List[String] = { + if (ttlMillis <= 0) return load + val now = System.currentTimeMillis() + val hit = cache.get(key) + if (hit != null && hit._1 > now) hit._2 + else { + val v = load + if (v.nonEmpty) cache.put(key, (now + ttlMillis, v)) + v + } + } } diff --git a/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilCacheTest.scala b/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilCacheTest.scala new file mode 100644 index 00000000..9cec173e --- /dev/null +++ b/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilCacheTest.scala @@ -0,0 +1,56 @@ +package org.sunbird.activity.util + +import java.util + +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.request.RequestContext +import org.sunbird.response.Response + +/** + * Tests the JVM-wide TTL cache added to HierarchyRelationsUtil.readFromDB: a repeated lookup for the + * same relationship_key is served from memory (DB hit once), and empty results are NOT cached (so a + * freshly-published collection is not held stale). Unique keys per test avoid cross-test cache bleed. + */ +class HierarchyRelationsUtilCacheTest extends AnyFlatSpec with Matchers with MockFactory { + + private def responseWith(nodeIds: util.List[String]): Response = { + val row = new util.HashMap[String, AnyRef]() {{ put("node_ids", nodeIds) }} + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(row) + val r = new Response(); r.put("response", rows); r + } + + private def nodeList(ids: String*): util.List[String] = { + val l = new util.ArrayList[String](); ids.foreach(l.add); l + } + + "getLeafNodes" should "hit the DB once and serve the repeat from cache" in { + val ops = mock[CassandraOperation] + // 4-arg getRecordsByProperties is what readFromDB uses; expect EXACTLY one DB call for two lookups + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(responseWith(nodeList("leaf-A", "leaf-B"))).once() + val util0 = HierarchyRelationsUtil(ops) + val col = "cacheHit-collection-unique-1" + val first = util0.getLeafNodes(col, col, null) + val second = util0.getLeafNodes(col, col, null) + first should contain allOf("leaf-A", "leaf-B") + second shouldBe first + } + + "readFromDB" should "not cache empty results (freshly-published collection is re-read)" in { + val ops = mock[CassandraOperation] + val emptyResp = new Response(); emptyResp.put("response", new util.ArrayList[util.Map[String, AnyRef]]()) + val populated = responseWith(nodeList("leaf-X")) + // first lookup empty (not published yet), second returns data -> BOTH must hit the DB + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(emptyResp).once() + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(populated).once() + val util0 = HierarchyRelationsUtil(ops) + val col = "negativeCache-collection-unique-2" + util0.getLeafNodes(col, col, null) shouldBe empty + util0.getLeafNodes(col, col, null) should contain("leaf-X") + } +} diff --git a/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTrackableTest.scala b/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTrackableTest.scala new file mode 100644 index 00000000..581242fa --- /dev/null +++ b/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTrackableTest.scala @@ -0,0 +1,40 @@ +package org.sunbird.activity.util + +import java.util + +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.request.RequestContext +import org.sunbird.response.Response + +/** getTrackableNodes reads the `::trackablenodes` relation, ordered, without dedup. */ +class HierarchyRelationsUtilTrackableTest extends AnyFlatSpec with Matchers with MockFactory { + + private def responseWith(nodeIds: util.List[String]): Response = { + val row = new util.HashMap[String, AnyRef]() {{ put("node_ids", nodeIds) }} + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(row) + val r = new Response(); r.put("response", rows); r + } + private def nodeList(ids: String*): util.List[String] = { + val l = new util.ArrayList[String](); ids.foreach(l.add); l + } + + "getTrackableNodes" should "return the ordered trackable ids for the root" in { + val ops = mock[CassandraOperation] + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(responseWith(nodeList("CRS-A", "CRS-B", "CRS-C"))).once() + val u = HierarchyRelationsUtil(ops) + u.getTrackableNodes("trk-root-unique-1", null) shouldBe List("CRS-A", "CRS-B", "CRS-C") + } + + it should "return empty when no trackablenodes relation exists" in { + val ops = mock[CassandraOperation] + val emptyResp = new Response(); emptyResp.put("response", new util.ArrayList[util.Map[String, AnyRef]]()) + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(emptyResp).once() + val u = HierarchyRelationsUtil(ops) + u.getTrackableNodes("trk-root-unique-2", null) shouldBe empty + } +} diff --git a/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala b/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala index 15cea294..9b17d106 100644 --- a/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala +++ b/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala @@ -18,9 +18,17 @@ class CassandraService(optionalDao: Option[CassandraOperation] = None) { private val activityTable = Option(ProjectUtil.getConfigValue("user_activity_agg_table")).getOrElse("user_activity_agg") private lazy val questionType: UserType = dao.getUDTType(keyspace, Option(ProjectUtil.getConfigValue("assessment_question_udt_type")).getOrElse("question")) + // assessment_aggregator identity columns are gated by viewer_enabled: viewer ON -> generalised names + // (table migrated), viewer OFF -> legacy names (un-migrated table). Lets the legacy /v1/assessment/agg + // path keep working on the old schema when the viewer is disabled. (user_activity_agg.context_id below + // is a DIFFERENT column and is NOT gated.) + private def viewerEnabled: Boolean = java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) + private def collectionCol: String = if (viewerEnabled) "collection_id" else "course_id" + private def contextCol: String = if (viewerEnabled) "context_id" else "batch_id" + def getAssessment(aid: String, uid: String, cid: String, bid: String, contId: String, ctx: RequestContext): Option[ExistingAssessment] = { try { - val filters = buildFilters("attempt_id" -> aid, "user_id" -> uid, "course_id" -> cid, "batch_id" -> bid, "content_id" -> contId) + val filters = buildFilters("attempt_id" -> aid, "user_id" -> uid, collectionCol -> cid, contextCol -> bid, "content_id" -> contId) val fields = java.util.Arrays.asList("attempt_id", "content_id", "last_attempted_on", "created_on", "total_score", "total_max_score", "question") val records = fetchRecords(filters, fields, ctx) records.headOption.map(mapToExisting) @@ -29,7 +37,7 @@ class CassandraService(optionalDao: Option[CassandraOperation] = None) { def getUserAssessments(uid: String, cid: String, bid: String, contId: String, ctx: RequestContext): List[ExistingAssessment] = { try { - val filters = buildFilters("user_id" -> uid, "course_id" -> cid, "batch_id" -> bid, "content_id" -> contId) + val filters = buildFilters("user_id" -> uid, collectionCol -> cid, contextCol -> bid, "content_id" -> contId) val fields = java.util.Arrays.asList("content_id", "attempt_id", "last_attempted_on", "total_max_score", "total_score", "question") fetchRecords(filters, fields, ctx).map(mapToExisting) } catch { case e: Throwable => logger.error(s"List failed for $uid", e); List.empty } @@ -77,8 +85,8 @@ class CassandraService(optionalDao: Option[CassandraOperation] = None) { rec.putAll(Map( "attempt_id" -> res.attemptId, "user_id" -> res.userId, - "course_id" -> res.courseId, - "batch_id" -> res.batchId, + collectionCol -> res.courseId, + contextCol -> res.batchId, "content_id" -> res.contentId, "total_score" -> res.totalScore.asInstanceOf[AnyRef], "total_max_score" -> res.totalMaxScore.asInstanceOf[AnyRef], diff --git a/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java b/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java index acd4c7eb..4f6a97c0 100644 --- a/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java +++ b/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java @@ -6,6 +6,8 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.sunbird.actor.base.BaseActor; +import org.sunbird.cassandra.CassandraOperation; +import org.sunbird.helper.ServiceFactory; import org.sunbird.common.ElasticSearchHelper; import org.sunbird.exception.ProjectCommonException; import org.sunbird.common.factory.EsClientFactory; @@ -43,6 +45,10 @@ public class CourseBatchManagementActor extends BaseActor { private CourseBatchDao courseBatchDao = new CourseBatchDaoImpl(); + private CassandraOperation cassandraOperation = ServiceFactory.getInstance(); + // Context key carrying the top-level (root) batch id down the auto-create recursion, so every + // chained descendant batch is one level under the root (`rootBatch:courseId`), never deep-chained. + private static final String ROOT_BATCH_ID = "rootBatchId"; private UserOrgService userOrgService = UserOrgServiceImpl.getInstance(); private UserCoursesService userCoursesService = new UserCoursesService(); private ElasticSearchService esService = EsClientFactory.getInstance(); @@ -82,7 +88,11 @@ private void createCourseBatch(Request actorMessage) throws Throwable { Map request = actorMessage.getRequest(); Map targetObject; List> correlatedObject = new ArrayList<>(); - String courseBatchId = ProjectUtil.getUniqueIdFromTimestamp(actorMessage.getEnv()); + // Use a client-supplied batchId when present (e.g. chained trackable-node batch ids); else generate. + String requestedBatchId = (String) request.get(JsonKey.BATCH_ID); + String courseBatchId = StringUtils.isNotBlank(requestedBatchId) + ? requestedBatchId + : ProjectUtil.getUniqueIdFromTimestamp(actorMessage.getEnv()); Map headers = (Map) actorMessage.getContext().get(JsonKey.HEADER); String requestedBy = (String) actorMessage.getContext().get(JsonKey.REQUESTED_BY); @@ -127,6 +137,76 @@ private void createCourseBatch(Request actorMessage) throws Throwable { if (courseNotificationActive()) { batchOperationNotifier(actorMessage, courseBatch, null); } + // A trackable collection whose trackablenodes contain other trackable collections auto-creates a + // batch for each trackable child by re-triggering this same actor; every child batch is one level + // under the root batch (`rootBatch:courseId`). A collection with no trackablenodes ⇒ no-op. + // Best-effort — the batch response is already sent above. + String rootBatchId = (String) actorMessage.getContext().getOrDefault(ROOT_BATCH_ID, courseBatchId); + triggerChildBatchCreation(actorMessage, courseId, rootBatchId); + } + + /** + * Fan out child-batch creation for a trackable collection. Reads {@code trackablenodes} for + * {@code parentCourseId}; for each trackable child, re-sends a {@code createBatch} to this actor + * with a flat root-prefixed batchId ({@code rootBatchId:childCourseId}). The root batch id is + * threaded through unchanged so deeper nesting stays one level (never {@code root:parent:child}). + * The existence guard makes it idempotent, so a child already reached via the flat trackablenodes + * list is not re-created when its parent recurses. + */ + private void triggerChildBatchCreation(Request parent, String parentCourseId, String rootBatchId) { + RequestContext ctx = parent.getRequestContext(); + try { + for (String childCourseId : getTrackableNodes(parentCourseId, ctx)) { + if (StringUtils.equalsIgnoreCase(childCourseId, parentCourseId)) continue; // guard: self-reference + String childBatchId = rootBatchId + ":" + childCourseId; + if (batchExists(childCourseId, childBatchId, ctx)) continue; // idempotent: safe re-run / gap-fill + Request child = new Request(ctx); + child.setOperation("createBatch"); + child.getContext().putAll(parent.getContext()); + child.getContext().put(ROOT_BATCH_ID, rootBatchId); // keep root prefix through recursion + Map childReq = new HashMap<>(parent.getRequest()); + childReq.put(JsonKey.COURSE_ID, childCourseId); + childReq.put(JsonKey.BATCH_ID, childBatchId); + // Child batches must NOT inherit the LP root's certificate template (that template is the LP cert). + // Course certs are controlled by `courseCertificates` (per-LP, default off); when enabled a course's + // own cert template is attached separately. Always strip the inherited template so the default is + // "LP cert only" and a course never wrongly issues the LP certificate. + childReq.remove("certTemplates"); + childReq.remove("cert_templates"); + child.setRequest(childReq); + logger.info(ctx, "triggerChildBatchCreation: creating batch " + childBatchId + " for course " + childCourseId); + self().tell(child, ActorRef.noSender()); + } + } catch (Exception ex) { + logger.error(ctx, "triggerChildBatchCreation failed for course=" + parentCourseId + ": " + ex.getMessage(), ex); + } + } + + /** Non-throwing existence check for a course_batch (readById throws when absent). */ + private boolean batchExists(String courseId, String batchId, RequestContext ctx) { + try { + courseBatchDao.readById(courseId, batchId, ctx); + return true; + } catch (ProjectCommonException e) { + return false; + } + } + + /** Reads the ordered trackable course ids from the `hierarchy_relations` row emitted at publish. */ + @SuppressWarnings("unchecked") + private List getTrackableNodes(String rootId, RequestContext ctx) { + String keyspace = Optional.ofNullable(ProjectUtil.getConfigValue("hierarchy_store_keyspace")) + .filter(StringUtils::isNotBlank).orElse("dev_hierarchy_store"); + String table = Optional.ofNullable(ProjectUtil.getConfigValue("hierarchy_relations_table")) + .filter(StringUtils::isNotBlank).orElse("hierarchy_relations"); + Map filters = new HashMap<>(); + filters.put("relationship_key", rootId + ":" + rootId + ":trackablenodes"); + List> rows = (List>) cassandraOperation + .getRecordsByProperties(keyspace, table, filters, ctx) + .getResult().getOrDefault(JsonKey.RESPONSE, new ArrayList<>()); + if (rows.isEmpty()) return Collections.emptyList(); + List nodeIds = (List) rows.get(0).get("node_ids"); + return nodeIds == null ? Collections.emptyList() : nodeIds; } private boolean courseNotificationActive() { diff --git a/modules/lms/course-mw/course-actors/src/test/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActorTest.java b/modules/lms/course-mw/course-actors/src/test/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActorTest.java index cc05cc26..7a07fd0b 100644 --- a/modules/lms/course-mw/course-actors/src/test/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActorTest.java +++ b/modules/lms/course-mw/course-actors/src/test/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActorTest.java @@ -3,13 +3,17 @@ import static org.powermock.api.mockito.PowerMockito.when; import org.apache.pekko.dispatch.Futures; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -138,6 +142,90 @@ public void getBatchSuccess() { Assert.assertNotNull(response); } + @Test + @PrepareForTest({ + ServiceFactory.class, + EsClientFactory.class, + UserOrgServiceImpl.class, + ContentUtil.class, InstructionEventGenerator.class, KafkaClient.class + }) + public void autoCreatesFlatChainedChildBatchesForTrackableNodes() throws Exception { + group = + MockerBuilder.getFreshMockerGroup() + .withCassandraMock(new CassandraMocker()) + .withESMock(new ESMocker()) + .withUserOrgMock(new UserOrgMocker()) + .andStaticMock(ContentUtil.class); + Map courseBatch = + CustomObjectBuilder.getCourseBatchBuilder().generateRandomFields().build().get(); + // Deterministic ids so we can assert the chained child batch ids. + courseBatch.put(JsonKey.COURSE_ID, "ROOT"); + courseBatch.put(JsonKey.BATCH_ID, "ROOT"); + + when(group.getESMockerService() + .save(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap(), Mockito.any())) + .thenReturn(Futures.successful("randomESID")); + when(ContentUtil.searchContent(Mockito.anyString(), Mockito.anyMap())) + .thenReturn(CustomObjectBuilder.getRandomCourse().get()); + + // trackablenodes read: ROOT has [C1, C2]; every other node has none (so recursion terminates). + Response trackable = new Response(); + Map row = new HashMap<>(); + row.put("node_ids", Arrays.asList("C1", "C2")); + trackable.put(JsonKey.RESPONSE, new ArrayList<>(Arrays.asList(row))); + Response emptyRel = new Response(); + emptyRel.put(JsonKey.RESPONSE, new ArrayList<>()); + when(group.getCassandraMockerService() + .getRecordsByProperties(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap(), Mockito.any())) + .thenAnswer(inv -> { + Map filters = inv.getArgument(2); + return "ROOT:ROOT:trackablenodes".equals(filters.get("relationship_key")) ? trackable : emptyRel; + }); + + // batchExists() existence check: no batch exists yet (readById throws on empty). + Response noBatch = new Response(); + noBatch.put(JsonKey.RESPONSE, new ArrayList<>()); + when(group.getCassandraMockerService() + .getRecordByIdentifier(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap(), Mockito.any(), Mockito.any())) + .thenReturn(noBatch); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(Map.class); + when(group.getCassandraMockerService() + .insertRecord(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap(), Mockito.any())) + .thenReturn(new CustomObjectBuilder.CustomObjectWrapper(true).asCassandraResponse()); + when(group.getUserOrgMockerService().getOrganisationById(Mockito.anyString())) + .thenReturn(CustomObjectBuilder.getRandomOrg().get()); + PowerMockito.mockStatic(InstructionEventGenerator.class); + PowerMockito.mockStatic(KafkaClient.class); + PowerMockito.doNothing().when(InstructionEventGenerator.class, "pushInstructionEvent", + Mockito.anyString(), Mockito.anyMap()); + PowerMockito.doNothing().when(KafkaClient.class, "send", Mockito.anyString(), Mockito.anyString()); + String orgId = ((List) courseBatch.get(JsonKey.COURSE_CREATED_FOR)).get(0); + when(group.getUserOrgMockerService().getUsersByIds(Mockito.anyList(), Mockito.anyString())) + .then((Answer>>) invocation -> + CustomObjectBuilder.getRandomUsersWithIds((List) invocation.getArguments()[0], orgId).get()); + when(group.getUserOrgMockerService().getUserById(Mockito.anyString(), Mockito.anyString())) + .then((Answer>) invocation -> + CustomObjectBuilder.getRandomUsersWithIds(Arrays.asList((String) invocation.getArguments()[0]), orgId).get().get(0)); + PowerMockito.mockStatic(ContentUtil.class); + mockCourseEnrollmentActor(); + + Request req = new Request(); + req.setOperation("createBatch"); + req.setRequest(courseBatch); + Response response = executeInTenSeconds(req, Response.class); + Assert.assertNotNull(response); + + // Root batch inserts synchronously; the two child batches are created via async self-messages. + Mockito.verify(group.getCassandraMockerService(), Mockito.timeout(8000).atLeast(3)) + .insertRecord(Mockito.anyString(), Mockito.anyString(), insertCaptor.capture(), Mockito.any()); + Set insertedValues = new HashSet<>(); + for (Map m : insertCaptor.getAllValues()) + for (Object v : m.values()) if (v instanceof String) insertedValues.add((String) v); + Assert.assertTrue("child batch ROOT:C1 not created", insertedValues.contains("ROOT:C1")); + Assert.assertTrue("child batch ROOT:C2 not created", insertedValues.contains("ROOT:C2")); + } + private void mockCourseEnrollmentActor(){ Map courseMap = new HashMap() {{ put("content", new HashMap() {{ diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala index 64665794..f4644db4 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala @@ -49,8 +49,13 @@ object AssessmentAuditRecorder { private def createRecordMap(m: util.Map[String, AnyRef], aid: String, uid: String, cid: String, ts: Long, ctx: RequestContext): util.Map[String, AnyRef] = { val rec = new util.HashMap[String, AnyRef]() - rec.put("user_id", uid); rec.put("course_id", m.get(JsonKey.COURSE_ID)) - rec.put("batch_id", m.get(JsonKey.BATCH_ID)); rec.put("content_id", cid) + // assessment_aggregator identity columns gated by viewer_enabled (new names when migrated / viewer on, + // legacy names when off) — keeps the legacy assessment path writing the un-migrated schema. + val viewerEnabled = java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) + rec.put("user_id", uid) + rec.put(if (viewerEnabled) "collection_id" else "course_id", m.get(JsonKey.COURSE_ID)) + rec.put(if (viewerEnabled) "context_id" else "batch_id", m.get(JsonKey.BATCH_ID)) + rec.put("content_id", cid) rec.put("attempt_id", aid) logger.info(ctx, s"AssessmentAuditRecorder: Recording attemptId=$aid with last_attempted_on=$ts") rec.put("last_attempted_on", new java.sql.Timestamp(ts)) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala index 988a5c0e..69ad0d65 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala @@ -14,6 +14,7 @@ import org.sunbird.response.ResponseCode import org.sunbird.utils.JsonUtil import org.sunbird.common.ProjectUtil import org.sunbird.helper.ServiceFactory +import org.sunbird.http.HttpClientUtil import org.sunbird.kafka.{InstructionEventGenerator, KafkaClient} import org.sunbird.learner.constants.{CourseJsonKey, InstructionEvent} import org.sunbird.learner.util.{CourseBatchUtil, Util} @@ -23,6 +24,10 @@ import java.util import java.util.{Date, TimeZone, UUID} import javax.inject.{Inject, Named} import org.apache.pekko.actor.ActorRef +import org.apache.pekko.pattern.ask +import org.apache.pekko.util.Timeout +import scala.concurrent.Await +import scala.concurrent.duration._ import scala.collection.JavaConverters._ import scala.collection.convert.ImplicitConversions._ @@ -99,8 +104,18 @@ class ContentConsumptionActor @Inject() ( } else contentList logger.info(requestContext, "Final content-consumption data: " + finalContentList) // Update consumption first and then push the assessment events if there are any. This will help us handling failures of max attempts (for assessment content). - val contentConsumptionResponse = processContents(finalContentList, requestContext, requestBy, requestedFor) - val assessmentResponse = processAssessments(assessmentEvents, requestContext, requestBy, requestedFor) + // viewer.enabled -> the Viewer Service OWNS both consumption AND assessment processing: + // regular contents -> /v1/view/start|end (ucc write + recursive rollup) + // assessment events -> /v1/assessment/submit (score + status=2 + same rollup) + // Assessments are therefore NOT merged into the content list here (submit marks completion + // itself, so no double rollup) and legacy processContents/processAssessments are skipped. + // API contract (per-key SUCCESS map) is preserved. + val contentConsumptionResponse = + if (isViewerEnabled) delegateContentsToViewer(contentList, request, requestBy, requestedFor) + else processContents(finalContentList, requestContext, requestBy, requestedFor) + val assessmentResponse = + if (isViewerEnabled) delegateAssessmentsToViewer(assessmentEvents, request, requestBy, requestedFor) + else processAssessments(assessmentEvents, requestContext, requestBy, requestedFor) val finalResponse = assessmentResponse.getOrElse(new Response()) finalResponse.putAll(contentConsumptionResponse.getOrElse(new Response()).getResult) sender().tell(finalResponse, self) @@ -429,13 +444,181 @@ class ContentConsumptionActor @Inject() ( activityAggregatorActor ! activityRequest } + // ProjectUtil.getConfigValue = env var first, then properties file. + private def isViewerEnabled: Boolean = + java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) + + // Transport for the viewer adapter: monolith -> in-JVM actor ask; distributed -> HTTP (default monolith). + private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) + private val viewerAskTimeoutMs: Long = + Option(ProjectUtil.getConfigValue("viewer_ask_timeout_ms")).filter(StringUtils.isNotBlank).map(_.trim.toLong).getOrElse(30000L) + private implicit val viewerAskTimeout: Timeout = Timeout(viewerAskTimeoutMs.millis) + private def viewerBaseUrl: String = + Option(ProjectUtil.getConfigValue("viewer_service_base_url")).filter(StringUtils.isNotBlank).getOrElse("http://viewer-service:9000") + private def viewerHeaders(token: String): java.util.Map[String, String] = + new java.util.HashMap[String, String]() {{ + put("Content-Type", "application/json") + Option(token).filter(StringUtils.isNotBlank).foreach(t => put("x-authenticated-user-token", t)) + }} + // Blocks until the viewer actor replies (rollup stays async inside the viewer); mirrors the old HTTP await. + private def viewerAsk(actorName: String, operation: String, body: java.util.Map[String, AnyRef], ctx: RequestContext): AnyRef = { + val req = new Request(); req.setRequestContext(ctx); req.setOperation(operation); req.setRequest(body) + Await.result(context.actorSelection("/user/" + actorName) ? req, viewerAskTimeoutMs.millis).asInstanceOf[AnyRef] + } + /** Write op (view/start|end, assessment/submit): monolith asks the in-JVM actor, distributed POSTs. */ + private def viewerWrite(actorName: String, httpApi: String, operation: String, + body: java.util.Map[String, AnyRef], token: String, ctx: RequestContext): Boolean = + if (isMonolith) viewerAsk(actorName, operation, body, ctx) != null + else StringUtils.isNotBlank(HttpClientUtil.post(viewerBaseUrl + httpApi, + mapper.writeValueAsString(new java.util.HashMap[String, AnyRef]() {{ put(JsonKey.REQUEST, body) }}), viewerHeaders(token), ctx)) + /** Read op (view/read): ucc rows. Monolith returns native Cassandra types; distributed parses JSON + coerces. */ + private def viewerRead(actorName: String, httpApi: String, operation: String, + body: java.util.Map[String, AnyRef], token: String, ctx: RequestContext): java.util.List[java.util.Map[String, AnyRef]] = { + val empty = new java.util.ArrayList[java.util.Map[String, AnyRef]]() + if (isMonolith) viewerAsk(actorName, operation, body, ctx) match { + case r: Response => r.getResult.getOrDefault(JsonKey.RESPONSE, empty).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + case _ => empty + } else { + val responseStr = HttpClientUtil.post(viewerBaseUrl + httpApi, + mapper.writeValueAsString(new java.util.HashMap[String, AnyRef]() {{ put(JsonKey.REQUEST, body) }}), viewerHeaders(token), ctx) + if (StringUtils.isBlank(responseStr)) empty + else { + val respNode = mapper.readTree(responseStr).path("result").path("response") + val rows = mapper.convertValue(respNode, classOf[java.util.List[java.util.Map[String, AnyRef]]]) + rows.asScala.foreach(coerceUccRowTypes) + rows + } + } + } + + /** + * Backward-compat adapter for content/state/update (content items): extract the fields and dispatch + * each to the viewer view API (status >= 2 -> viewEnd, else viewStart). The viewer owns the ucc write + * + async rollup. Idempotent monotonic upserts, so retries are safe. Response: contentId -> SUCCESS/FAILED. + */ + private def delegateContentsToViewer(contentList: java.util.List[java.util.Map[String, AnyRef]], + originalRequest: Request, + requestedBy: String, requestedFor: String): Option[Response] = { + if (CollectionUtils.isEmpty(contentList)) return None + val ctx = originalRequest.getRequestContext + val userId = if (StringUtils.isNotBlank(requestedFor)) requestedFor else requestedBy + val token = originalRequest.getContext.get(JsonKey.X_AUTH_TOKEN).asInstanceOf[String] + val responseMessage = new java.util.HashMap[String, AnyRef]() + contentList.asScala.foreach(c => { + val contentId = c.get(JsonKey.CONTENT_ID).asInstanceOf[String] + try { + val status = c.getOrDefault(JsonKey.STATUS, 0.asInstanceOf[AnyRef]).asInstanceOf[Number].intValue() + val collectionId = Option(c.get(JsonKey.COLLECTION_ID)).getOrElse(c.get(JsonKey.COURSE_ID)).asInstanceOf[String] + val (op, api) = if (status >= 2) ("viewEnd", "/v1/view/end") else ("viewStart", "/v1/view/start") + val body = new java.util.HashMap[String, AnyRef]() {{ + put("contentId", contentId) + put("collectionId", collectionId) + put("contextId", c.get(JsonKey.BATCH_ID)) + put(JsonKey.BATCH_ID, c.get(JsonKey.BATCH_ID)) + put(JsonKey.USER_ID, userId) + Option(c.get("progressdetails")).orElse(Option(c.get("progressDetails"))).foreach(pd => put("progressDetails", pd)) + }} + responseMessage.put(contentId, if (viewerWrite("view-consumption-actor", api, op, body, token, ctx)) JsonKey.SUCCESS else "FAILED") + } catch { + case ex: Exception => + logger.error(ctx, s"delegateContentsToViewer failed for contentId=$contentId: ${ex.getMessage}", ex) + responseMessage.put(contentId, "FAILED") + } + }) + val response = new Response(); response.putAll(responseMessage); Option(response) + } + + /** + * Backward-compat adapter for content/state/update (assessment events): dispatch each to the viewer + * assessment/submit (score + status=2 + rollup). courseId/batchId map to collectionId/contextId. + * Response: batchId -> SUCCESS/FAILED. + */ + private def delegateAssessmentsToViewer(assessmentEvents: java.util.List[java.util.Map[String, AnyRef]], + originalRequest: Request, + requestedBy: String, requestedFor: String): Option[Response] = { + if (CollectionUtils.isEmpty(assessmentEvents)) return None + val ctx = originalRequest.getRequestContext + val userId = if (StringUtils.isNotBlank(requestedFor)) requestedFor else requestedBy + val token = originalRequest.getContext.get(JsonKey.X_AUTH_TOKEN).asInstanceOf[String] + val responseMessage = new java.util.HashMap[String, AnyRef]() + assessmentEvents.asScala.foreach(a => { + val batchId = a.getOrDefault(JsonKey.BATCH_ID, "").asInstanceOf[String] + try { + val collectionId = Option(a.get(JsonKey.COLLECTION_ID)).getOrElse(a.get(JsonKey.COURSE_ID)).asInstanceOf[String] + val events = a.getOrDefault(JsonKey.ASSESSMENT_EVENTS_KEY, new java.util.ArrayList[java.util.Map[String, AnyRef]]()) + val body = new java.util.HashMap[String, AnyRef]() {{ + put("contentId", a.get(JsonKey.CONTENT_ID)) + put("collectionId", collectionId) + put("contextId", batchId) + put(JsonKey.USER_ID, userId) + put(JsonKey.ASSESSMENT_EVENTS, events) + }} + responseMessage.put(batchId, if (viewerWrite("view-consumption-actor", "/v1/assessment/submit", "viewAssess", body, token, ctx)) JsonKey.SUCCESS else "FAILED") + } catch { + case ex: Exception => + logger.error(ctx, s"delegateAssessmentsToViewer failed for batchId=$batchId: ${ex.getMessage}", ex) + responseMessage.put(batchId, "FAILED") + } + }) + val response = new Response(); response.putAll(responseMessage); Option(response) + } + + /** + * Backward-compat adapter for content/state/read: fetch the ucc rows from the viewer (viewRead) instead + * of a direct Cassandra read. Returns the same row shape getContentsConsumption produces (monolith: + * native Cassandra types; distributed: coerced back from JSON), so the caller's post-processing is unchanged. + */ + private def readContentsFromViewer(userId: String, courseId: String, batchId: String, + contentIds: java.util.List[String], originalRequest: Request): java.util.List[java.util.Map[String, AnyRef]] = { + val ctx = originalRequest.getRequestContext + val token = originalRequest.getContext.get(JsonKey.X_AUTH_TOKEN).asInstanceOf[String] + val body = new java.util.HashMap[String, AnyRef]() {{ + put(JsonKey.USER_ID, userId) + put("collectionId", courseId) + put("contextId", batchId) + if (CollectionUtils.isNotEmpty(contentIds)) put("contentId", contentIds) + }} + try viewerRead("view-consumption-actor", "/v1/view/read", "viewRead", body, token, ctx) + catch { + case ex: Exception => + logger.error(ctx, s"readContentsFromViewer failed for userId=$userId courseId=$courseId: ${ex.getMessage}", ex) + new java.util.ArrayList[java.util.Map[String, AnyRef]]() + } + } + + // ucc column -> native type, so JSON-parsed viewer rows match a direct Cassandra read exactly. + private val uccTimestampCols = Set("last_access_time", "last_completed_time", "last_updated_time", "datetime") + private val uccIntCols = Set("status", "progress", "completedcount", "viewcount") + + private def coerceUccRowTypes(row: java.util.Map[String, AnyRef]): Unit = { + uccTimestampCols.foreach { c => val d = toDate(row.get(c)); if (d != null) row.put(c, d) } + uccIntCols.foreach { c => row.get(c) match { case n: Number => row.put(c, Integer.valueOf(n.intValue())); case _ => } } + row.get("completionpercentage") match { case n: Number => row.put("completionpercentage", java.lang.Float.valueOf(n.floatValue())); case _ => } + } + + /** Reconstruct a java.util.Date from the viewer's JSON form (epoch-millis number, numeric string, or ISO-8601). */ + private def toDate(v: AnyRef): java.util.Date = v match { + case null => null + case d: java.util.Date => d + case n: Number => new java.util.Date(n.longValue()) + case s: String if StringUtils.isNotBlank(s) => + try new java.util.Date(s.toLong) + catch { case _: Throwable => try java.util.Date.from(java.time.Instant.parse(s)) catch { case _: Throwable => null } } + case _ => null + } + def getConsumption(request: Request): Unit = { val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] val batchId = request.get(JsonKey.BATCH_ID).asInstanceOf[String] val courseId = request.get(JsonKey.COURSE_ID).asInstanceOf[String] val contentIds = request.getRequest.getOrDefault(JsonKey.CONTENT_IDS, new java.util.ArrayList[String]()).asInstanceOf[java.util.List[String]] val fields = request.getRequest.getOrDefault(JsonKey.FIELDS, new java.util.ArrayList[String](){{ add(JsonKey.PROGRESS) }}).asInstanceOf[java.util.List[String]] - val contentsConsumed = getContentsConsumption(userId, courseId, contentIds, batchId, request.getRequestContext) + // viewer.enabled -> Content-State-Read is served by the Viewer Service (/v1/view/read), so raw + // ucc reads can be archived away from this API. Same rows, so the existing post-processing + // (field filtering, json parse, date format, optional assessment score) is reused unchanged. + val contentsConsumed = + if (isViewerEnabled) readContentsFromViewer(userId, courseId, batchId, contentIds, request) + else getContentsConsumption(userId, courseId, contentIds, batchId, request.getRequestContext) val response = new Response if(CollectionUtils.isNotEmpty(contentsConsumed)) { val filteredContents = contentsConsumed.map(m => { @@ -468,8 +651,9 @@ class ContentConsumptionActor @Inject() ( val filters = new java.util.HashMap[String, AnyRef]() { { put("user_id", userId) - put("course_id", courseId) - put("batch_id", batchId) + // gated by viewer_enabled: new names when the assessment table is migrated, legacy otherwise + put(if (isViewerEnabled) "collection_id" else "course_id", courseId) + put(if (isViewerEnabled) "context_id" else "batch_id", batchId) put("content_id", contentId) } } diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index cb25608d..11f2b7f5 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -44,6 +44,9 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c var courseBatchDao: CourseBatchDao = new CourseBatchDaoImpl() var userCoursesDao: UserCoursesDao = new UserCoursesDaoImpl() var groupDao: GroupDaoImpl = new GroupDaoImpl() + private lazy val cassandraOperation = org.sunbird.helper.ServiceFactory.getInstance + private val jsonMapper = new ObjectMapper() + private def isViewerEnabled: Boolean = java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) private val redisEnabled: Boolean = RedisCacheUtil.isRedisEnabled val isCacheEnabled = redisEnabled && (if (StringUtils.isNotBlank(ProjectUtil.getConfigValue("user_enrolments_response_cache_enable"))) (ProjectUtil.getConfigValue("user_enrolments_response_cache_enable")).toBoolean else true) @@ -74,6 +77,7 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c request.getOperation match { case "enrol" => enroll(request) + case "systemEnrol" => systemEnroll(request) case "unenrol" => unEnroll(request) case "listEnrol" => list(request) case _ => ProjectCommonException.throwClientErrorException(ResponseCode.invalidRequestData, @@ -90,6 +94,9 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c validateEnrolment(batchData, enrolmentData, true) val data: java.util.Map[String, AnyRef] = createUserEnrolmentMap(userId, courseId, batchId, enrolmentData, request.getContext.getOrDefault(JsonKey.REQUEST_ID, "").asInstanceOf[String]) upsertEnrollment(userId, courseId, batchId, data, (null == enrolmentData), request.getRequestContext) + // viewer.enabled: also enrol the trackable descendant nodes (best-effort, never fails the root enrol) + if (isViewerEnabled) + enrolTrackableDescendants(userId, courseId, batchId, request.getContext.getOrDefault(JsonKey.REQUEST_ID, "").asInstanceOf[String], request.getRequestContext) if (isCacheEnabled) { logger.info(request.getRequestContext, "CourseEnrolmentActor :: enroll :: Deleting redis for key " + getCacheKey(userId)) cacheUtil.delete(getCacheKey(userId)) @@ -98,6 +105,28 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c generateTelemetryAudit(userId, courseId, batchId, data, "enrol", JsonKey.CREATE, request.getContext) notifyUser(userId, batchData, JsonKey.ADD) } + + /** + * System-driven enrol (the `doEnrol` seam) — used by LP progression to open a course. + * Reuses the SAME verified write path as `enroll` (createUserEnrolmentMap + upsertEnrollment + + * cache-clear + telemetry audit), so LP auto-enrolments are first-class. Differences: `addedBy = + * system-lp`, notifications SUPPRESSED (no per-auto-enrol spam), no descendant fan-out (this is a + * single course), and idempotent (already-enrolled -> success no-op). Called via the + * ProgressionEnroller gateway: in-JVM (monolith) or HTTP (distributed). + */ + def systemEnroll(request: Request): Unit = { + val courseId: String = request.get(JsonKey.COURSE_ID).asInstanceOf[String] + val userId: String = request.get(JsonKey.USER_ID).asInstanceOf[String] + val batchId: String = request.get(JsonKey.BATCH_ID).asInstanceOf[String] + val enrolmentData: UserCourses = userCoursesDao.read(request.getRequestContext, userId, courseId, batchId) + if (null != enrolmentData) { sender().tell(successResponse(), self); return } // idempotent + val data: java.util.Map[String, AnyRef] = createUserEnrolmentMap(userId, courseId, batchId, enrolmentData, "system-lp") + upsertEnrollment(userId, courseId, batchId, data, true, request.getRequestContext) + if (isCacheEnabled) cacheUtil.delete(getCacheKey(userId)) + sender().tell(successResponse(), self) + generateTelemetryAudit(userId, courseId, batchId, data, "enrol", JsonKey.CREATE, request.getContext) + // notifications intentionally suppressed for system-lp; no enrolTrackableDescendants (single course). + } def unEnroll(request:Request): Unit = { @@ -263,6 +292,62 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c } }} + /** + * viewer.enabled: on root enrol, also create a user_enrolments row for every TRACKABLE descendant + * (trackable.enabled == "Yes") so nested-node progress has an enrolment to land in. Batch ids are + * chained to the nearest trackable ancestor (`parentBatch:nodeId`); the matching course_batch rows + * are created out of band (manual / batch-create API with an explicit batchId). Best-effort: any + * failure is logged, never fails the root enrol. Creates NO batches and skips non-trackable nodes. + * ponytail: reads full hierarchy JSON per enrol, no cache — add a TTL cache if enrol throughput needs it. + */ + private def enrolTrackableDescendants(userId: String, rootId: String, rootBatchId: String, requestedBy: String, ctx: RequestContext): Unit = { + try { + val hierarchy = readCollectionHierarchy(rootId, ctx) + if (hierarchy == null) { logger.info(ctx, s"enrolTrackableDescendants: no hierarchy for $rootId"); return } + val acc = scala.collection.mutable.ListBuffer[(String, String)]() + collectTrackable(hierarchy, rootBatchId, acc) + acc.foreach { case (nodeId, nodeBatch) => + if (null == userCoursesDao.read(ctx, userId, nodeId, nodeBatch)) { + val data = createUserEnrolmentMap(userId, nodeId, nodeBatch, null, requestedBy) + upsertEnrollment(userId, nodeId, nodeBatch, data, true, ctx) + logger.info(ctx, s"enrolTrackableDescendants: enrolled node=$nodeId batch=$nodeBatch user=$userId") + } + } + } catch { + case ex: Exception => logger.error(ctx, s"enrolTrackableDescendants failed root=$rootId user=$userId: ${ex.getMessage}", ex) + } + } + + private def readCollectionHierarchy(rootId: String, ctx: RequestContext): java.util.Map[String, AnyRef] = { + val keyspace = Option(ProjectUtil.getConfigValue("hierarchy_store_keyspace")).filter(StringUtils.isNotBlank).getOrElse("dev_hierarchy_store") + val table = Option(ProjectUtil.getConfigValue("content_hierarchy_table")).filter(StringUtils.isNotBlank).getOrElse("content_hierarchy") + val filters = new java.util.HashMap[String, AnyRef]() {{ put("identifier", rootId) }} + val rows = cassandraOperation.getRecordsByProperties(keyspace, table, filters.asInstanceOf[java.util.Map[String, AnyRef]], ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new java.util.ArrayList[java.util.Map[String, AnyRef]]) + .asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + if (rows.isEmpty) return null + val json = rows.get(0).get("hierarchy").asInstanceOf[String] + if (StringUtils.isBlank(json)) null else jsonMapper.readValue(json, classOf[java.util.Map[String, AnyRef]]) + } + + /** Recurse children: a trackable node -> (id, parentBatch:id) and becomes the parent batch for its subtree; non-trackable nodes are transparent structure. */ + private def collectTrackable(node: java.util.Map[String, AnyRef], effParentBatch: String, acc: scala.collection.mutable.ListBuffer[(String, String)]): Unit = { + val children = node.get("children") + if (children == null) return + children.asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]].asScala.foreach { child => + val id = child.get("identifier").asInstanceOf[String] + val trackable = child.get("trackable").asInstanceOf[java.util.Map[String, AnyRef]] + val enabled = trackable != null && "Yes".equalsIgnoreCase(String.valueOf(trackable.get("enabled"))) + if (enabled && StringUtils.isNotBlank(id)) { + val nodeBatch = effParentBatch + ":" + id + acc += ((id, nodeBatch)) + collectTrackable(child, nodeBatch, acc) + } else { + collectTrackable(child, effParentBatch, acc) + } + } + } + def notifyUser(userId: String, batchData: CourseBatch, operationType: String): Unit = { val isNotifyUser = java.lang.Boolean.parseBoolean(PropertiesCache.getInstance().getProperty(JsonKey.SUNBIRD_COURSE_BATCH_NOTIFICATIONS_ENABLED)) if(isNotifyUser){ diff --git a/modules/viewer/actors/pom.xml b/modules/viewer/actors/pom.xml new file mode 100644 index 00000000..4c57e555 --- /dev/null +++ b/modules/viewer/actors/pom.xml @@ -0,0 +1,175 @@ + + + + + org.sunbird + viewer + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + viewer-actors + Viewer Actors + Viewer actors: granular view lifecycle + recursive collection tracking + + + 11 + 11 + 2.13 + 2.13.12 + 1.0.3 + UTF-8 + + + + + + org.scala-lang + scala-library + ${scala.maj.version} + + + org.apache.pekko + pekko-actor_${scala.version} + ${pekko.version} + + + + + org.sunbird + course-actors-common + 1.0-SNAPSHOT + + + org.sunbird + sunbird-actor-utils + 1.0-SNAPSHOT + + + org.sunbird + enrolment-actor + 1.0-SNAPSHOT + + + org.sunbird + sunbird-cache-utils + + + + + + org.sunbird + activity-aggregator + 1.0-SNAPSHOT + + + org.sunbird + assessment-aggregator + 1.0-SNAPSHOT + + + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.commons + commons-lang3 + 3.12.0 + + + org.apache.commons + commons-collections4 + 4.4 + + + + + org.scalatest + scalatest_${scala.version} + 3.2.15 + test + + + org.scalamock + scalamock_${scala.version} + 5.2.0 + test + + + org.apache.pekko + pekko-testkit_${scala.version} + ${pekko.version} + test + + + + + src/main/scala + src/test/scala + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + ${scala.maj.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + + org.scalatest + scalatest-maven-plugin + 2.0.0 + + + test + test + + test + + + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar + + + + + + + diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala new file mode 100644 index 00000000..d08a1e36 --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala @@ -0,0 +1,266 @@ +package org.sunbird.viewer.actor + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.commons.collections4.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.pekko.actor.ActorRef +import org.sunbird.assessment.models._ +import org.sunbird.assessment.service.{AssessmentService, CassandraService, ContentService} +import org.sunbird.assessment.util.AssessmentParser +import org.sunbird.common.ProjectUtil +import org.sunbird.enrolments.BaseEnrolmentActor +import org.sunbird.helper.ServiceFactory +import org.sunbird.keys.JsonKey +import org.sunbird.learner.util.Util +import org.sunbird.request.{Request, RequestContext} +import org.sunbird.response.Response + +import java.util +import javax.inject.{Inject, Named} +import scala.collection.JavaConverters._ + +/** + * Granular view lifecycle writes to user_content_consumption (ucc). + * + * Write model = read-modify-upsert with MONOTONIC merge (mirrors ContentConsumptionActor; + * NOT Paxos LWT). Race-free by monotonicity + Cassandra per-cell LWW + per-userId serialization: + * viewStart -> INSERT only if absent (status 1); if present, no-op. + * viewUpdate -> merge only if row exists; revisits (already status 2) ignored. + * viewEnd -> status 2 + completed time; then async aggregation (fire-and-forget tell). + * + * ucc PK (viewer schema §2): (userid, collectionid, contextid, contentid). + * No collection context -> collectionid = contextid = contentid. + */ +class ViewConsumptionActor @Inject() ( + @Named("viewer-aggregator-actor") viewerAggregatorActor: ActorRef +) extends BaseEnrolmentActor { + + private val mapper = new ObjectMapper + private var cassandraOperation = ServiceFactory.getInstance + private val consumptionDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_CONTENT_DB) + private val CONSUMPTION_TABLE = "user_content_consumption" + + // Assessment scoring reuses the assessment-aggregator services in-process (same math + persistence + // the legacy AssessmentAggregatorActor uses). ContentService is only touched if metadata validation + // is invoked — the viewer submit path does not call it, so no content-search network hop. + private lazy val assessmentService = new AssessmentService(new ContentService()) + // built on the same injected CassandraOperation, so setCassandraOperation() also controls the + // assessment persistence in tests (and keeps a single Cassandra handle in production). + private lazy val assessmentCassandra = new CassandraService(Some(cassandraOperation)) + + override def onReceive(request: Request): Unit = { + request.getOperation match { + case "viewStart" => viewStart(request) + case "viewUpdate" => viewUpdate(request) + case "viewEnd" => viewEnd(request) + case "viewRead" => viewRead(request) + case "viewAssess" => viewAssess(request) + case "assessmentRead" => assessmentRead(request) + case _ => onReceiveUnsupportedOperation(request.getOperation) + } + } + + /** + * /v1/assessment/submit (api.view.assess). Scores the attempt (reuse AssessmentService + + * CassandraService — identical to legacy AssessmentAggregatorActor), then treats the assessment + * like a completed content: mark ucc status=2 and run the same sync rollup as viewEnd, so the + * assessment leaf counts toward collection completion. Score aggregates (score:cid/max_score:cid) + * land in user_activity_agg via putAll append; the rollup's completion agg uses different keys → + * they coexist. Request: userId, collectionId?, contextId?, contentId, assessments[] (assess events). + */ + private def viewAssess(request: Request): Unit = { + val ctx = request.getRequestContext + val key = viewKey(request) + val userId = key.get("userid").asInstanceOf[String] + val collectionId = key.get("collectionid").asInstanceOf[String] + val contextId = key.get("contextid").asInstanceOf[String] + val contentId = key.get("contentid").asInstanceOf[String] + + val eventsRaw = Option(request.get(JsonKey.ASSESSMENT_EVENTS)).orElse(Option(request.get(JsonKey.EVENTS))) + .map(_.asInstanceOf[util.List[util.Map[String, AnyRef]]]).getOrElse(new util.ArrayList[util.Map[String, AnyRef]]()) + val events: List[AssessmentEvent] = eventsRaw.asScala.map(AssessmentParser.mapToEvent).toList + + if (events.nonEmpty) { + val ts = Option(request.get("assessmentTs")).orElse(Option(request.get("assessmentTimestamp"))) + .map(_.asInstanceOf[Number].longValue()).getOrElse(System.currentTimeMillis()) + val attemptId = Option(request.get(JsonKey.ATTEMPT_ID)).map(_.toString).filter(StringUtils.isNotBlank) + .getOrElse(java.util.UUID.randomUUID().toString) // no client attemptId -> a fresh attempt (avoids hashCode collisions overwriting a prior attempt) + val unique = assessmentService.getUniqueQuestions(events) + val metrics = assessmentService.computeScoreMetrics(unique) + val result = AssessmentResult(attemptId, userId, collectionId, contextId, contentId, + metrics.totalScore, metrics.totalMaxScore, metrics.grandTotal, metrics.questions, System.currentTimeMillis(), ts) + assessmentCassandra.saveAssessment(result, ctx) + // best-score across all attempts -> user_activity_agg (reuse legacy aggregation) + val stored = assessmentCassandra.getUserAssessments(userId, collectionId, contextId, contentId, ctx) + val agg = assessmentService.computeUserAggregates(userId, collectionId, contextId, stored) + assessmentCassandra.updateUserActivity(userId, collectionId, contextId, agg, ctx) + } else { + logger.warn(ctx, s"viewAssess: no assessment events for userId=$userId contentId=$contentId; marking complete only", null) + } + + // Assessment content is complete on submit (score-independent, mirrors legacy status=2) -> ucc + rollup. + val row = new util.HashMap[String, AnyRef](key) + row.put("status", Integer.valueOf(2)) + row.put("last_completed_time", ProjectUtil.getTimeStamp) + row.put("last_updated_time", ProjectUtil.getTimeStamp) + cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + triggerAggregation(request, ctx) + + val out = new Response(); out.put(contentId, JsonKey.SUCCESS); sender().tell(out, self) + } + + /** + * /v1/assessment/read (api.assessment.read). Best score / max score per content from + * assessment_aggregator (reuse getUserAssessments). Request: userId, contentId[] , collectionId?, contextId?. + */ + private def assessmentRead(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] + val collectionId = ViewerRequestKeys.collectionId(request).orNull + val contextId = ViewerRequestKeys.contextId(request).orNull + val contentIds: List[String] = request.get("contentId") match { + case l: util.List[_] => l.asScala.map(_.asInstanceOf[String]).toList + case s: String if StringUtils.isNotBlank(s) => List(s) + case _ => List.empty + } + val contents = new util.ArrayList[util.Map[String, AnyRef]]() + contentIds.foreach { cid => + val stored = assessmentCassandra.getUserAssessments(userId, collectionId, contextId, cid, ctx) + if (stored.nonEmpty) { + val best = stored.maxBy(_.totalScore) + val m = new util.HashMap[String, AnyRef]() + m.put("identifier", cid) + m.put("score", best.totalScore.asInstanceOf[AnyRef]) + m.put("max_score", best.totalMaxScore.asInstanceOf[AnyRef]) + contents.add(m) + } + } + val out = new Response() + out.put(JsonKey.USER_ID, userId) + out.put("collectionId", collectionId) + out.put("contextId", contextId) + out.put("contents", contents) + sender().tell(out, self) + } + + /** Raw ucc rows for a user's content(s) under a collection. context=all -> ignore contextid. */ + private def viewRead(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] + val allContexts = "all".equalsIgnoreCase(request.get("context").asInstanceOf[String]) + val filters = new util.HashMap[String, AnyRef]() + filters.put("userid", userId) + ViewerRequestKeys.collectionId(request).foreach(c => filters.put("collectionid", c)) + if (!allContexts) ViewerRequestKeys.contextId(request).foreach(c => filters.put("contextid", c)) + val contentIds = request.get("contentId") match { + case l: util.List[_] => l.asScala.map(_.asInstanceOf[String]).asJava + case s: String if StringUtils.isNotBlank(s) => util.Arrays.asList(s) + case _ => null + } + if (contentIds != null && !contentIds.isEmpty) filters.put("contentid", contentIds) + val response = cassandraOperation.getRecords(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + val rows = response.getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + val out = new Response(); out.put(JsonKey.RESPONSE, rows); sender().tell(out, self) + } + + private def viewStart(request: Request): Unit = { + val ctx = request.getRequestContext + val key = viewKey(request) + val existing = readRow(key, ctx) + if (existing == null) { + val row = new util.HashMap[String, AnyRef](key) + row.put("status", Integer.valueOf(1)) + row.put("last_access_time", ProjectUtil.getTimeStamp) + row.put("last_updated_time", ProjectUtil.getTimeStamp) + cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + } + // present -> already started, no-op + sender().tell(successResponse(), self) + } + + private def viewUpdate(request: Request): Unit = { + val ctx = request.getRequestContext + val key = viewKey(request) + val existing = readRow(key, ctx) + if (existing != null && statusOf(existing) < 2) { + val row = new util.HashMap[String, AnyRef](key) + // status is monotonic; an update never downgrades and never completes (that's viewEnd) + row.put("status", Integer.valueOf(math.max(1, statusOf(existing)))) + Option(request.get("progressDetails")).foreach(pd => row.put("progressdetails", mapper.writeValueAsString(pd))) + row.put("last_access_time", ProjectUtil.getTimeStamp) + row.put("last_updated_time", ProjectUtil.getTimeStamp) + cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + } + // absent -> update only if exists (ignore); already completed -> revisit ignored + sender().tell(successResponse(), self) + } + + private def viewEnd(request: Request): Unit = { + val ctx = request.getRequestContext + val key = viewKey(request) + val row = new util.HashMap[String, AnyRef](key) + row.put("status", Integer.valueOf(2)) + Option(request.get("progressDetails")).foreach(pd => row.put("progressdetails", mapper.writeValueAsString(pd))) + row.put("last_completed_time", ProjectUtil.getTimeStamp) + row.put("last_updated_time", ProjectUtil.getTimeStamp) + cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + // Async rollup: fire-and-forget tell to the aggregator; respond immediately (does not wait). + triggerAggregation(request, ctx) + sender().tell(successResponse(), self) + } + + /** Fire-and-forget tell to ViewerAggregatorActor; the rollup runs async — the response does not wait. */ + private def triggerAggregation(request: Request, ctx: RequestContext): Unit = { + val key = viewKey(request) + val aggRequest = new Request() + aggRequest.setOperation("aggregate") + aggRequest.setRequestContext(ctx) + aggRequest.put(JsonKey.USER_ID, key.get("userid")) + aggRequest.put("collectionId", key.get("collectionid")) + aggRequest.put(JsonKey.BATCH_ID, key.get("contextid")) + // Async, fire-and-forget: the rollup + LP progression run in the background on the aggregator + // (per-user serialized). The hot path does not wait for it — the change from before is ask -> tell. + viewerAggregatorActor.tell(aggRequest, ActorRef.noSender) + } + + /** + * Build the ucc primary key (live column names userid, collectionid, contextid, contentid). + * Backward-compatible request keys: collectionId (else legacy courseId), contextId (else legacy + * batchId). No collection ctx -> collectionid = contextid = contentId. + */ + private def viewKey(request: Request): util.HashMap[String, AnyRef] = { + // explicit userId (internal delegation) else requestedFor/requestedBy (from token on direct API calls) + val userId = Option(request.get(JsonKey.USER_ID).asInstanceOf[String]).filter(StringUtils.isNotBlank) + .orElse(Option(request.get(JsonKey.REQUESTED_FOR).asInstanceOf[String]).filter(StringUtils.isNotBlank)) + .getOrElse(request.get(JsonKey.REQUESTED_BY).asInstanceOf[String]) + val contentId = ViewerRequestKeys.contentId(request) + val collectionId = ViewerRequestKeys.collectionId(request).getOrElse(contentId) + val contextId = ViewerRequestKeys.contextId(request).getOrElse(contentId) + val key = new util.HashMap[String, AnyRef]() + key.put("userid", userId) + key.put("collectionid", collectionId) + key.put("contextid", contextId) + key.put("contentid", contentId) + key + } + + private def readRow(key: util.HashMap[String, AnyRef], ctx: RequestContext): util.Map[String, AnyRef] = { + val filters = new util.HashMap[String, AnyRef](key) + val response = cassandraOperation.getRecords(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + val rows = response.getResult + .getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) rows.get(0) else null + } + + private def statusOf(row: util.Map[String, AnyRef]): Int = + Option(row.get("status")).map(_.asInstanceOf[Number].intValue()).getOrElse(0) + + // for tests + def setCassandraOperation(ops: org.sunbird.cassandra.CassandraOperation): ViewConsumptionActor = { + cassandraOperation = ops + this + } +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala new file mode 100644 index 00000000..c0552a38 --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -0,0 +1,378 @@ +package org.sunbird.viewer.actor + +import org.apache.commons.collections4.CollectionUtils +import org.sunbird.activity.domain.{ContentStatus, UserContentConsumption, UserEnrolmentAgg} +import org.sunbird.activity.util.{ActivityAggregateUtil, CertificateUtil, HierarchyRelationsUtil} +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.common.ProjectUtil +import org.sunbird.http.HttpClientUtil +import org.sunbird.enrolments.BaseEnrolmentActor +import org.sunbird.helper.ServiceFactory +import org.sunbird.keys.JsonKey +import org.sunbird.learner.util.Util +import org.sunbird.request.{Request, RequestContext} +import org.sunbird.viewer.util.ProgressionPolicy + +import java.util +import scala.collection.JavaConverters._ + +/** + * Sync, in-request recursive rollup for the viewer module. Invoked from viewEnd (per-userId serialized). + * + * REUSE: all aggregation math is ActivityAggregateUtil (same calls ActivityAggregatorActor uses). + * The util treats collectionId as the activity_id slot and batchId as the context slot — it does not + * care about the names. Viewer deltas vs ActivityAggregatorActor: + * - optionality is PER-USER: `optional_nodes` from user_enrolments (NOT hierarchy getOptionalNodes). + * required = collectionLeafNodes.diff(userOpt) at leaf AND every ancestor level. + * - recompute from DB state each call -> idempotent, safe under per-user serialization. + * + * ASSUMPTIONS to verify against live schema (v2 snake_case): + * - user_content_consumption PK (user_id, collection_id, context_id, content_id); status per content. + * - user_enrolments keyed (userid, courseid, batchid) with optional_nodes set. + * - user_activity_agg is the aggregate target (activity_id = collection do-id). + */ +class ViewerAggregatorActor extends BaseEnrolmentActor { + + private var cassandraOperation: CassandraOperation = ServiceFactory.getInstance + private var hierarchyRelationsUtil: HierarchyRelationsUtil = HierarchyRelationsUtil(cassandraOperation) + private var certificateUtil: CertificateUtil = CertificateUtil() + private val activityAggUtil = new ActivityAggregateUtil() + + private val enrolmentDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_COURSE_DB) + private val activityAggDBInfo = Util.dbInfoMap.get(JsonKey.GROUP_ACTIVITY_DB) + private val consumptionDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_CONTENT_DB) + private val CONSUMPTION_TABLE = "user_content_consumption" + + override def onReceive(request: Request): Unit = { + request.getOperation match { + case "aggregate" => + // Fire-and-forget rollup: log failures (caller already acked); /v1/view/agg force-sync is the repair path. + try aggregate(request) + catch { case ex: Exception => logger.error(request.getRequestContext, s"ViewerAggregatorActor.aggregate failed: ${ex.getMessage}", ex) } + sender().tell(successResponse(), self) + case _ => onReceiveUnsupportedOperation(request.getOperation) + } + } + + private def aggregate(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] + // accept collectionId (else legacy courseId) / contextId (else legacy batchId) + val collectionId = ViewerRequestKeys.collectionId(request).orNull + val batchId = ViewerRequestKeys.contextId(request).orNull + if (userId == null || collectionId == null) { + logger.warn(ctx, s"ViewerAggregatorActor: missing userId/collectionId, skipping", null) + return + } + + // trackablenodes non-empty => this root is a Learning Path (structural detection; §Step 2/5). + val trackable = hierarchyRelationsUtil.getTrackableNodes(collectionId, ctx) + + // 1. Read this user's consumption for the (root) collection+context from viewer ucc, build status map + val rows = readConsumption(userId, collectionId, batchId, ctx) + if (CollectionUtils.isEmpty(rows)) { + // No consumption yet. For an LP, still advance (bootstrap: open the first required course). + if (trackable.nonEmpty) advanceLp(userId, collectionId, batchId, trackable, ctx) + else logger.info(ctx, s"ViewerAggregatorActor: no consumption for userId=$userId collectionId=$collectionId") + return + } + val contentStatusMap: Map[String, ContentStatus] = activityAggUtil.getContentStatusFromContents(rows) + val uc = UserContentConsumption(userId, batchId, collectionId, contentStatusMap) + + // 2. Per-learner optional COURSES from user_enrolments.optional_nodes (LP policy; course-level). + val perLearnerOptionalCourses: List[String] = readOptionalNodes(userId, collectionId, batchId, ctx) + + // 3. Root leaves + the tree's nodes (via ancestors) — needed before computing effectiveOptional. + val leafNodes = hierarchyRelationsUtil.getLeafNodes(collectionId, collectionId, ctx) + if (leafNodes.isEmpty) { + logger.warn(ctx, s"ViewerAggregatorActor: no leafNodes for collectionId=$collectionId; is hierarchy_relations published?", null) + return + } + val ancestors: Map[String, List[String]] = uc.contents.map { case (contentId, content) => + (contentId, hierarchyRelationsUtil.getAncestors(collectionId, content.contentId, ctx)) + }.toMap + val childCollections = ancestors.values.flatten.filter(_ != collectionId).toList.distinct + + // effectiveOptional LEAVES (§5.1) = author-marked hierarchy `optionalnodes` (content-level, all nodes) + // ∪ leaves of per-learner optional courses (course-level, expanded to leaves so the leaf-vs-leaf diff works). + val treeNodes = collectionId :: childCollections + val hierarchyOptionalLeaves = treeNodes.flatMap(n => hierarchyRelationsUtil.getOptionalNodes(collectionId, n, ctx)).distinct + val optionalCourseLeaves = perLearnerOptionalCourses.flatMap(c => hierarchyRelationsUtil.getLeafNodes(collectionId, c, ctx)).distinct + val effectiveOptional: List[String] = (hierarchyOptionalLeaves ++ optionalCourseLeaves).distinct + + // 4. Aggregates: root + every ancestor node; required per node = its leafNodes − effectiveOptional. + val courseAgg = activityAggUtil.computeCourseActivityAgg(uc, leafNodes, effectiveOptional, ctx) + val collectionsWithLeafNodes: Map[String, List[String]] = childCollections.map { col => + (col, hierarchyRelationsUtil.getLeafNodes(collectionId, col, ctx).diff(effectiveOptional)) + }.toMap + val moduleAggs = activityAggUtil.computeModuleActivityAgg(uc, collectionId, ancestors, collectionsWithLeafNodes, ctx) + + val allAggs: List[UserEnrolmentAgg] = courseAgg.toList ++ moduleAggs + + // 5. Write user_activity_agg (frozen content_status + agg) for root + every node + writeActivityAggregates(allAggs, ctx) + + // 6. Per-node progress: nodeId -> (completedCount, requiredLeaves) for root + every trackable ancestor. + val nodeProgress = scala.collection.mutable.LinkedHashMap[String, (Int, List[String])]() + courseAgg.foreach(a => nodeProgress(collectionId) = (completedCountOf(a), leafNodes.diff(effectiveOptional))) + moduleAggs.foreach(a => nodeProgress(a.activityAgg.activity_id) = + (completedCountOf(a), collectionsWithLeafNodes.getOrElse(a.activityAgg.activity_id, Nil))) + + // 7. Update user_enrolments status for EVERY enrolled node in this tree (approach #1: key off the + // child enrolment rows that already exist; root included). Cert fires once, on transition to complete. + writeAllNodeEnrolments(userId, collectionId, batchId, nodeProgress.toMap, contentStatusMap, ctx) + + // 8. LP progression (only when this root is an LP): optionality once, open next course(s), credit at completion. + if (trackable.nonEmpty) advanceLp(userId, collectionId, batchId, trackable, ctx) + } + + private def completedCountOf(a: UserEnrolmentAgg): Int = + a.activityAgg.aggregates.getOrElse("completedCount", 0.0).toInt + + // ─────────────────────────── LP progression (the engine) ─────────────────────────── + // Pure decisions come from ProgressionPolicy; this orchestrates reads/writes. Strict is fully + // functional. Adaptive/PriorLearning are wired but their skill inputs (policy source, se_skills, + // diagnostic assessment scores) are marked VERIFY-ON-DEPLOY — they default to "no skills" so the + // system compiles and behaves as Strict until those integrations are wired against the live env. + + private val USER_SKILLS_TABLE = "user_skills" + + private def advanceLp(userId: String, rootId: String, batchId: String, + trackable: List[String], ctx: RequestContext): Unit = { + ensureOptionalityComputed(userId, rootId, batchId, trackable, ctx) + val optional = readOptionalNodes(userId, rootId, batchId, ctx).toSet + val ancestorsOf = (n: String) => hierarchyRelationsUtil.getAncestors(rootId, n, ctx) + val levels = ProgressionPolicy.orderedLevels(trackable, ancestorsOf, rootId) + + // Level complete = all its required (non-optional) courses complete (empty required set = complete, §5). + // Derived from persisted enrolment status only, so it's recompute-safe (force-sync repairs identically). + def courseComplete(c: String): Boolean = enrolStatus(userId, c, batchId + ":" + c, ctx).contains(2) + def levelComplete(level: String): Boolean = + ProgressionPolicy.coursesOfLevel(level, trackable, ancestorsOf, rootId).filterNot(optional.contains).forall(courseComplete) + + // Open first incomplete level: enrol all its optionals up front + the next single required course (§5). + levels.find(l => !levelComplete(l)).foreach { level => + val courses = ProgressionPolicy.coursesOfLevel(level, trackable, ancestorsOf, rootId) + val nextRequired = courses.filterNot(optional.contains).find(c => !courseComplete(c)) + (courses.filter(optional.contains) ++ nextRequired.toList).foreach { c => + val childBatch = batchId + ":" + c + if (!isEnrolled(userId, c, childBatch, ctx)) internalEnrol(userId, c, childBatch, ctx) + } + } + + // LP completion = every level complete -> credit durable skills (once; creditSkills no-ops if nothing new). + if (levels.nonEmpty && levels.forall(levelComplete)) creditSkills(userId, rootId, trackable, ctx) + } + + private def ensureOptionalityComputed(userId: String, rootId: String, batchId: String, trackable: List[String], ctx: RequestContext): Unit = { + // Compute once (§Step 4); optionalityComputed remembers empty results without a DB column. + if (optionalityComputed(userId, rootId, batchId, ctx)) return + val policy = policyOf(rootId, ctx) + if (policy.equalsIgnoreCase("Strict") || trackable.isEmpty) { writeOptionalNodes(userId, rootId, batchId, Set.empty, ctx); return } + val diagnostic = trackable.head + val hasDiagnostic = isAssessmentCourse(diagnostic, ctx) + if (hasDiagnostic && !isComplete(userId, diagnostic, batchId, ctx)) return // wait for the diagnostic + val prior = if (policy.equalsIgnoreCase("PriorLearning")) readUserSkills(userId, ctx) else Set.empty[String] + val fromDiag = if (hasDiagnostic) skillsFromAssessment(userId, rootId, diagnostic, ctx) else Set.empty[String] + val achieved = prior ++ fromDiag + val meta = courseMeta(trackable, ctx) + val assessmentCourses = meta.collect { case (c, (_, true)) => c }.toSet + val skillsByCourse = meta.map { case (c, (s, _)) => c -> s } + writeOptionalNodes(userId, rootId, batchId, + ProgressionPolicy.computeOptionalNodes(policy, trackable, skillsByCourse, assessmentCourses, achieved), ctx) + } + + // VERIFY-ON-DEPLOY: policy source. Read the LP's policy from collection/batch metadata; absent => Strict. + private def policyOf(rootId: String, ctx: RequestContext): String = "Strict" + // VERIFY-ON-DEPLOY: assessment detection. Needs a Practice-Question-Set child via /v3/search or content_hierarchy. + private def isAssessmentCourse(courseId: String, ctx: RequestContext): Boolean = false + // VERIFY-ON-DEPLOY: per-course skills (se_skills) + assessment flag via /v3/search. + private def courseMeta(trackable: List[String], ctx: RequestContext): Map[String, (Set[String], Boolean)] = + trackable.map(c => c -> (Set.empty[String], isAssessmentCourse(c, ctx))).toMap + // VERIFY-ON-DEPLOY: best-attempt assessment_aggregator scores × se_skills tags -> ProgressionPolicy.computeAchievedSkills. + private def skillsFromAssessment(userId: String, rootId: String, courseId: String, ctx: RequestContext): Set[String] = Set.empty + + private def isComplete(userId: String, courseId: String, rootBatchId: String, ctx: RequestContext): Boolean = + enrolStatus(userId, courseId, rootBatchId + ":" + courseId, ctx).contains(2) + + private def enrolStatus(userId: String, collectionId: String, contextId: String, ctx: RequestContext): Option[Int] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("collectionid", collectionId); put("contextid", contextId) }} + val rows = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) Option(rows.get(0).get("status")).map(_.asInstanceOf[Number].intValue()) else None + } + + private def isEnrolled(userId: String, collectionId: String, contextId: String, ctx: RequestContext): Boolean = + enrolStatus(userId, collectionId, contextId, ctx).isDefined + + /** + * Internal (system-driven) enrol via the ProgressionEnroller gateway — the FULL enrol op (`doEnrol`), + * NOT a bare DAO write: it reuses CourseEnrolmentActor's verified write path (DB + cache + telemetry). + * Transport per `deployment_mode`: MONOLITH -> in-JVM message to the enrolment actor's `systemEnrol`; + * DISTRIBUTED -> HTTP to the enrolment service. Idempotent (`systemEnrol` no-ops if already enrolled). + */ + private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) // default monolith + + private def internalEnrol(userId: String, collectionId: String, contextId: String, ctx: RequestContext): Unit = { + if (isMonolith) { + val req = new Request() + req.setRequestContext(ctx) + req.setOperation("systemEnrol") + req.put(JsonKey.USER_ID, userId); req.put(JsonKey.COURSE_ID, collectionId); req.put(JsonKey.BATCH_ID, contextId) + // VERIFY-ON-DEPLOY: bound path of the enrolment actor in the monolith actor system. + val path = Option(ProjectUtil.getConfigValue("enrolment_actor_path")).filter(_.nonEmpty).getOrElse("/user/course-enrolment-actor") + // noSender: fire-and-forget; the enrol actor's success reply must NOT bounce back to this actor + // (it only handles "aggregate" Requests) — let the reply go to deadLetters. + context.actorSelection(path).tell(req, org.apache.pekko.actor.ActorRef.noSender) + } else { + // DISTRIBUTED: call the enrolment service over HTTP (full enrol op). + // VERIFY-ON-DEPLOY: use a system-enrol endpoint (not the public one that fans out/notifies) + forward auth token. + val base = Option(ProjectUtil.getConfigValue("enrolment_service_base_url")).filter(_.nonEmpty).getOrElse("http://lern-service:9000") + val body = s"""{"request":{"userId":"$userId","courseId":"$collectionId","batchId":"$contextId"}}""" + val headers = new util.HashMap[String, String]() {{ + put("Content-Type", "application/json") + // System-driven enrol: authenticate with the configured system token (else 401 in distributed mode). + Option(ProjectUtil.getConfigValue("viewer_system_auth_token")).filter(_.nonEmpty) + .foreach(t => put("x-authenticated-user-token", t)) + }} + HttpClientUtil.post(base + "/v1/course/enroll", body, headers, ctx) + } + logger.info(ctx, s"ViewerAggregatorActor: system-enrol requested course=$collectionId ctx=$contextId user=$userId mode=${ProjectUtil.getConfigValue("deployment_mode")}") + } + + private def writeOptionalNodes(userId: String, rootId: String, batchId: String, optional: Set[String], ctx: RequestContext): Unit = { + val selectMap = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("collectionid", rootId); put("contextid", batchId) }} + val updateMap = new util.HashMap[String, AnyRef]() {{ put("optional_nodes", optional.asJava) }} + cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) + ViewerAggregatorActor.markOptionalityComputed(userId, rootId, batchId) // remember empty results too (no DB column) + } + + /** Computed? Non-empty optional_nodes is self-evident; an empty result is remembered in an in-process memo. */ + private def optionalityComputed(userId: String, rootId: String, batchId: String, ctx: RequestContext): Boolean = + readOptionalNodes(userId, rootId, batchId, ctx).nonEmpty || + ViewerAggregatorActor.isOptionalityComputed(userId, rootId, batchId) + + private def readUserSkills(userId: String, ctx: RequestContext): Set[String] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} + val rows = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, USER_SKILLS_TABLE, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) + Option(rows.get(0).get("skills")).map(_.asInstanceOf[util.Collection[String]].asScala.toSet).getOrElse(Set.empty) + else Set.empty + } + + // Durable skill credit — ONCE at LP completion. Read-union-upsert (portable; no set-append needed). + private def creditSkills(userId: String, rootId: String, trackable: List[String], ctx: RequestContext): Unit = { + val earned = trackable.filter(c => isAssessmentCourse(c, ctx)).flatMap(c => skillsFromAssessment(userId, rootId, c, ctx)).toSet + if (earned.isEmpty) return // no-op until se_skills is wired (VERIFY-ON-DEPLOY) + val existing = readUserSkills(userId, ctx) + val merged = existing ++ earned + if (merged.size == existing.size) return // nothing new — credit already granted (advanceLp calls this every completed pass) + val row = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("skills", merged.asJava) }} + cassandraOperation.insertRecord(enrolmentDBInfo.getKeySpace, USER_SKILLS_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + logger.info(ctx, s"ViewerAggregatorActor: credited ${earned.size} skills to user=$userId for LP=$rootId") + } + + private def writeActivityAggregates(aggs: List[UserEnrolmentAgg], ctx: RequestContext): Unit = { + val aggQueries = aggs.map(a => activityAggUtil.createActivityAggUpdateMap(a.activityAgg)).asJava + if (!aggQueries.isEmpty) + cassandraOperation.batchUpdateWithPutAll(activityAggDBInfo.getKeySpace, activityAggDBInfo.getTableName, aggQueries, ctx) + } + + /** + * Approach #1: update user_enrolments status for every node in this tree that has an enrolment row. + * Matches each row on collectionid ∈ tree AND this LP's contextid (§4: standalone enrolments untouched). + * Cert fires once, only on the transition to complete (status != 2 -> 2). + */ + private def writeAllNodeEnrolments(userId: String, rootId: String, batchId: String, + nodeProgress: Map[String, (Int, List[String])], + contentStatusMap: Map[String, ContentStatus], ctx: RequestContext): Unit = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} + val enrolRows = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + enrolRows.asScala.foreach { row => + val nodeId = Option(row.get("collectionid")).map(_.toString).orNull + val nodeCtx = Option(row.get("contextid")).map(_.toString).orNull + // This LP only (root=batchId, child=batchId:childId); a standalone enrolment's contextid differs (§4). + val expectedCtx = if (nodeId == rootId) batchId else batchId + ":" + nodeId + nodeProgress.get(nodeId).filter(_ => nodeCtx == expectedCtx).foreach { case (completedCount, requiredLeaves) => + val required = requiredLeaves.size + val status = activityAggUtil.getCompletionStatus(completedCount, required) + val pct = activityAggUtil.getCompletionPercentage(completedCount, required) + val currentStatus = Option(row.get("status")).map(_.asInstanceOf[Number].intValue()).getOrElse(0) + val nodeContentStatus = requiredLeaves.flatMap(l => contentStatusMap.get(l).map(cs => l -> Integer.valueOf(cs.status))).toMap + val selectMap = new util.HashMap[String, AnyRef]() {{ + put("userid", userId); put("collectionid", nodeId); put("contextid", nodeCtx) + }} + val updateMap = new util.HashMap[String, AnyRef]() {{ + put("progress", Integer.valueOf(completedCount)) + put("status", Integer.valueOf(status)) + put("completionpercentage", Integer.valueOf(pct)) + put("contentstatus", nodeContentStatus.asJava) + if (status == 2 && currentStatus != 2) put("completedon", new java.util.Date()) + }} + cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) + if (status == 2 && currentStatus != 2) { + logger.info(ctx, s"ViewerAggregatorActor: node completed userId=$userId collectionId=$nodeId; issuing cert") + certificateUtil.publishCertificateIssueEvent(userId, nodeId, nodeCtx, ctx) + } + } + } + } + + /** + * Read this user's ucc rows for the collection, scoped to the context. + * (userid, collectionid, contextid) is a clustering-prefix slice on PK + * (userid, collectionid, contextid, contentid) -> efficient, no scan. + * contextId omitted only when absent (no-context viewer), falling back to collection-wide read. + */ + private def readConsumption(userId: String, collectionId: String, contextId: String, ctx: RequestContext): util.List[util.Map[String, AnyRef]] = { + val filters = new util.HashMap[String, AnyRef]() {{ + put("userid", userId) + put("collectionid", collectionId) + if (contextId != null) put("contextid", contextId) + }} + val response = cassandraOperation.getRecords(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + response.getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + } + + /** Per-user optional_nodes from user_enrolments (empty for strict policy). */ + private def readOptionalNodes(userId: String, collectionId: String, batchId: String, ctx: RequestContext): List[String] = { + val filters = new util.HashMap[String, AnyRef]() {{ + put("userid", userId) + put("collectionid", collectionId) + put("contextid", batchId) + }} + val response = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + val rows = response.getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) { + Option(rows.get(0).get("optional_nodes")) + .map(_.asInstanceOf[util.Collection[String]].asScala.toList) + .getOrElse(List()) + } else List() + } + + // for tests + def configure(ops: CassandraOperation, hru: HierarchyRelationsUtil, cu: CertificateUtil): ViewerAggregatorActor = { + cassandraOperation = ops; hierarchyRelationsUtil = hru; certificateUtil = cu; this + } +} + +object ViewerAggregatorActor { + // JVM-wide memo of enrolments whose (empty) LP optionality is computed, so we don't recompute each pass. + // ponytail: unbounded set, entries live for the process lifetime; add a size cap / TTL only if it grows. + private val optionalityDone: java.util.Set[String] = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() + private def optKey(userId: String, rootId: String, batchId: String): String = s"$userId:$rootId:$batchId" + def markOptionalityComputed(userId: String, rootId: String, batchId: String): Unit = + optionalityDone.add(optKey(userId, rootId, batchId)) + def isOptionalityComputed(userId: String, rootId: String, batchId: String): Boolean = + optionalityDone.contains(optKey(userId, rootId, batchId)) +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala new file mode 100644 index 00000000..8d75c5ae --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala @@ -0,0 +1,31 @@ +package org.sunbird.viewer.actor + +import org.apache.commons.lang3.StringUtils +import org.sunbird.request.Request + +/** + * Backward-compatible request-key resolution for the viewer APIs. + * + * The viewer generalises course -> collection. Clients (and the content-state delegation) may send + * the OLD keys courseId/batchId or the NEW keys collectionId/contextId — both are accepted, mapped + * to the collection/context concept. camelCase (API convention) with lowercase fallbacks. + * This is the REQUEST-payload layer only; DB columns are handled separately. + */ +object ViewerRequestKeys { + + private def firstNonBlank(request: Request, keys: String*): Option[String] = + keys.iterator + .map(k => request.get(k)) + .collectFirst { case v: String if StringUtils.isNotBlank(v) => v } + + /** collectionId, else legacy courseId. */ + def collectionId(request: Request): Option[String] = + firstNonBlank(request, "collectionId", "collectionid", "courseId", "courseid") + + /** contextId, else legacy batchId. */ + def contextId(request: Request): Option[String] = + firstNonBlank(request, "contextId", "contextid", "batchId", "batchid") + + def contentId(request: Request): String = + firstNonBlank(request, "contentId", "contentid").orNull +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala new file mode 100644 index 00000000..ca4a7b0a --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala @@ -0,0 +1,141 @@ +package org.sunbird.viewer.actor + +import org.apache.commons.collections4.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.sunbird.enrolments.BaseEnrolmentActor +import org.sunbird.helper.ServiceFactory +import org.sunbird.keys.JsonKey +import org.sunbird.learner.util.Util +import org.sunbird.request.{Request, RequestContext} +import org.sunbird.response.Response + +import java.util +import scala.collection.JavaConverters._ + +/** + * Summary APIs for the viewer module — enrolment-level (distinct from the per-content view actor). + * Reads/deletes user_enrolments only; never writes consumption. Raw ucc reads (viewRead) live on + * ViewConsumptionActor. + * + * summaryRead -> per-enrolment progress/status/contentstatus for one collection + * summaryList -> all enrolment summaries for a user + * summaryDelete -> delete enrolment rows (all, or one collection+context) + */ +class ViewerSummaryActor extends BaseEnrolmentActor { + + private var cassandraOperation = ServiceFactory.getInstance + private val enrolmentDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_COURSE_DB) + + override def onReceive(request: Request): Unit = { + request.getOperation match { + case "summaryRead" => summaryRead(request) + case "summaryList" => summaryList(request) + case "summaryDownload" => summaryDownload(request) + case "summaryDelete" => summaryDelete(request) + case _ => onReceiveUnsupportedOperation(request.getOperation) + } + } + + /** Per-enrolment progress/status from user_enrolments (identified by collectionId). */ + private def summaryRead(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] + val collectionId = ViewerRequestKeys.collectionId(request).orNull + val batchId = ViewerRequestKeys.contextId(request).orNull + + // Identified by collectionId (courseid). user_enrolments carries progress/status/completionpercentage + // + per-content contentstatus for the enrolment — no activity_type needed. + val enrolFilters = new util.HashMap[String, AnyRef]() + enrolFilters.put("userid", userId) + if (StringUtils.isNotBlank(collectionId)) enrolFilters.put("collectionid", collectionId) + if (StringUtils.isNotBlank(batchId)) enrolFilters.put("contextid", batchId) + val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, enrolFilters, ctx) + + val response = new Response + response.put(JsonKey.RESPONSE, enrolments) + sender().tell(response, self) + } + + /** All enrolment summaries for a user (partition by user_id). */ + private def summaryList(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = Option(request.get(JsonKey.USER_ID).asInstanceOf[String]) + .getOrElse(request.get("userId").asInstanceOf[String]) + val filters = new util.HashMap[String, AnyRef]() + filters.put("userid", userId) + val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters, ctx) + val response = new Response + response.put(JsonKey.RESPONSE, enrolments) + sender().tell(response, self) + } + + /** + * Exhaust download of a user's enrolment summaries. format=json (default) returns the rows; + * format=csv returns a CSV string under "content". ponytail: inline export (no cloud upload / signed + * URL) — fine for per-user summaries; switch to cloud-storage-sdk + a returned URL if exhaust grows + * large or needs a stored artifact. + */ + private def summaryDownload(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = Option(request.get(JsonKey.USER_ID).asInstanceOf[String]) + .getOrElse(request.get("userId").asInstanceOf[String]) + val format = Option(request.get("format").asInstanceOf[String]).map(_.toLowerCase).getOrElse("json") + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} + val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters, ctx) + val response = new Response + response.put("format", format) + if (format == "csv") response.put("content", toCsv(enrolments)) + else response.put(JsonKey.RESPONSE, enrolments) + sender().tell(response, self) + } + + private val csvCols = List("collectionid", "contextid", "progress", "status", "completionpercentage", "completedon") + private def toCsv(rows: util.List[util.Map[String, AnyRef]]): String = { + val sb = new StringBuilder(csvCols.mkString(",")).append("\n") + rows.asScala.foreach { r => + sb.append(csvCols.map(c => Option(r.get(c)).map(_.toString.replace(",", " ")).getOrElse("")).mkString(",")).append("\n") + } + sb.toString + } + + /** Delete enrolment rows: all for the user, or a single collection[+batch]. */ + private def summaryDelete(request: Request): Unit = { + val ctx = request.getRequestContext + val userId = Option(request.get(JsonKey.USER_ID).asInstanceOf[String]) + .getOrElse(request.get("userId").asInstanceOf[String]) + val collectionId = ViewerRequestKeys.collectionId(request).orNull + val batchId = ViewerRequestKeys.contextId(request).orNull + + if (StringUtils.isBlank(collectionId)) { + // delete all: fetch keys then delete each row + val rows = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, + new util.HashMap[String, AnyRef]() {{ put("userid", userId) }}, ctx) + rows.asScala.foreach(r => deleteEnrolment(userId, strOrNull(r.get("collectionid")), strOrNull(r.get("contextid")), ctx)) + } else { + deleteEnrolment(userId, collectionId, batchId, ctx) + } + sender().tell(successResponse(), self) + } + + private def deleteEnrolment(userId: String, collectionId: String, batchId: String, ctx: RequestContext): Unit = { + val key = new util.HashMap[String, String]() + key.put("userid", userId) + if (StringUtils.isNotBlank(collectionId)) key.put("collectionid", collectionId) + if (StringUtils.isNotBlank(batchId)) key.put("contextid", batchId) + cassandraOperation.deleteRecord(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, key, ctx) + } + + private def getRecords(keyspace: String, table: String, filters: util.HashMap[String, AnyRef], ctx: RequestContext): util.List[util.Map[String, AnyRef]] = { + val response = cassandraOperation.getRecords(keyspace, table, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + response.getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) + .asInstanceOf[util.List[util.Map[String, AnyRef]]] + } + + private def strOrNull(v: AnyRef): String = if (v == null) null else v.asInstanceOf[String] + + // for tests + def setCassandraOperation(ops: org.sunbird.cassandra.CassandraOperation): ViewerSummaryActor = { + cassandraOperation = ops + this + } +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala new file mode 100644 index 00000000..5d035a0c --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala @@ -0,0 +1,90 @@ +package org.sunbird.viewer.util + +import scala.jdk.CollectionConverters._ + +/** + * Pure, host-agnostic decisions for Learning-Path progression. No I/O — every input is passed in, + * so these are trivially unit-testable and run identically in-request or in the async aggregator. + * (Structural helpers only for now; level/optionality helpers are added in a later slice.) + */ +object ProgressionPolicy { + + private val ASSESSMENT_CATEGORY = "practice question set" + + private def isTrackable(node: java.util.Map[String, AnyRef]): Boolean = + node.get("trackable") match { + case t: java.util.Map[_, _] => + "Yes".equalsIgnoreCase(String.valueOf(t.asInstanceOf[java.util.Map[String, AnyRef]].get("enabled"))) + case _ => false + } + + private def childrenOf(node: java.util.Map[String, AnyRef]): List[java.util.Map[String, AnyRef]] = + node.get("children") match { + case l: java.util.List[_] => l.asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]].asScala.toList + case _ => Nil + } + + /** + * Structural LP detection: true iff the collection has a descendant that is itself a trackable + * collection (`trackable.enabled == "Yes"`). A plain course — whose children are non-trackable + * content — is false. No reliance on `policy`/`primaryCategory`. + */ + def hasNestedTrackable(rootNode: java.util.Map[String, AnyRef]): Boolean = { + def hasTrackableDescendant(node: java.util.Map[String, AnyRef]): Boolean = + childrenOf(node).exists(c => isTrackable(c) || hasTrackableDescendant(c)) + hasTrackableDescendant(rootNode) + } + + /** A course is an assessment course iff it has a child with `primaryCategory == "Practice Question Set"`. */ + def isAssessment(courseNode: java.util.Map[String, AnyRef]): Boolean = + childrenOf(courseNode).exists(c => + ASSESSMENT_CATEGORY.equalsIgnoreCase(String.valueOf(c.get("primaryCategory")))) + + // ── Level helpers + optionality resolver (pure; take their data as parameters, no I/O) ── + + /** + * The course's level = the ancestor that is a **direct child of the root** (top-most under root), + * i.e. `lastOption` of the ancestor chain excluding the root — NOT the nearest ancestor. For a + * 2-deep root->level->course tree they coincide; only this definition is correct if a course is + * nested deeper inside a level. `ancestorsOf` returns the chain nearest-first. + */ + def levelOf(course: String, ancestorsOf: String => List[String], root: String): Option[String] = + ancestorsOf(course).filterNot(_ == root).lastOption + + /** Trackable courses whose level is `level`, preserving `trackable` (trackablenodes) order. */ + def coursesOfLevel(level: String, trackable: List[String], + ancestorsOf: String => List[String], root: String): List[String] = + trackable.filter(c => levelOf(c, ancestorsOf, root).contains(level)) + + /** Distinct level nodes in first-appearance order along `trackable`. */ + def orderedLevels(trackable: List[String], + ancestorsOf: String => List[String], root: String): List[String] = + trackable.flatMap(c => levelOf(c, ancestorsOf, root)).distinct + + /** + * A course is optional iff it is not an assessment and all of its (non-empty) skills are achieved. + * `Strict` waives nothing. `Adaptive`/`PriorLearning` differ only in how `skillsAchieved` is built + * by the caller — this function is policy-agnostic beyond the `Strict` short-circuit. + */ + def computeOptionalNodes(policy: String, courses: List[String], + skillsByCourse: Map[String, Set[String]], + assessmentCourses: Set[String], + skillsAchieved: Set[String]): Set[String] = { + if ("Strict".equalsIgnoreCase(policy)) Set.empty + else courses.filter { c => + val skills = skillsByCourse.getOrElse(c, Set.empty) + !assessmentCourses.contains(c) && skills.nonEmpty && skills.subsetOf(skillsAchieved) + }.toSet + } + + /** + * A skill is achieved iff **all** of its tagged questions are correct. Pure core of `skillsFrom` + * (the aggregator supplies `skillToQuestions` from `/v3/search` tags and `correctQuestions` from + * `assessment_aggregator`). Skills with no tagged questions are never achieved. + */ + def computeAchievedSkills(skillToQuestions: Map[String, Set[String]], + correctQuestions: Set[String]): Set[String] = + skillToQuestions.collect { + case (skill, qs) if qs.nonEmpty && qs.subsetOf(correctQuestions) => skill + }.toSet +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala new file mode 100644 index 00000000..f403767d --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala @@ -0,0 +1,145 @@ +package org.sunbird.viewer.actor + +import java.util +import java.util.concurrent.TimeUnit + +import org.apache.pekko.actor.{Actor, ActorSystem, Props} +import org.apache.pekko.testkit.TestKit +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.request.{Request, RequestContext} +import org.sunbird.response.Response + +import scala.concurrent.ExecutionContext +import scala.concurrent.duration.FiniteDuration + +/** + * Unit tests for ViewConsumptionActor — view lifecycle (start/update/end/read) + assessment + * (submit no-events branch + read). CassandraOperation is mocked via the setCassandraOperation seam; + * the aggregator is a stub actor that replies immediately so the sync ask in viewEnd/viewAssess + * returns fast. Scoring math itself is covered by AssessmentServiceSpec, not re-tested here. + */ +class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactory { + + implicit val ec: ExecutionContext = ExecutionContext.global + val system: ActorSystem = ActorSystem.create("viewer-consumption-test") + + // stub aggregator: replies to the Patterns.ask so triggerAggregation completes without the 30s timeout + private def replyingAggregator = system.actorOf(Props(new Actor { + def receive: Receive = { case _ => sender() ! new Response() } + })) + + private def emptyRows: Response = { + val r = new Response(); r.put("response", new util.ArrayList[util.Map[String, AnyRef]]()); r + } + + private def rowsWith(rows: util.List[util.Map[String, AnyRef]]): Response = { + val r = new Response(); r.put("response", rows); r + } + + private def uccRow(status: Int): util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]() {{ + put("userid", "u1"); put("collectionid", "c1"); put("contextid", "b1"); put("contentid", "ct1") + put("status", Integer.valueOf(status)) + }} + + private def callActor(request: Request, props: Props): Response = { + val probe = new TestKit(system) + val actorRef = system.actorOf(props) + actorRef.tell(request, probe.testActor) + probe.expectMsgType[Response](FiniteDuration.apply(15, TimeUnit.SECONDS)) + } + + private def viewRequest(op: String): Request = { + val req = new Request + req.setOperation(op) + req.put("userId", "u1"); req.put("collectionId", "c1"); req.put("contextId", "b1"); req.put("contentId", "ct1") + req + } + + "viewStart" should "insert a new ucc row when absent" in { + val ops = mock[CassandraOperation] + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows) + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + val result = callActor(viewRequest("viewStart"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + "viewStart" should "be a no-op when the row already exists" in { + val ops = mock[CassandraOperation] + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(uccRow(1)) + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(rowsWith(rows)) + // no upsertRecord expectation -> a call would fail the strict mock + val result = callActor(viewRequest("viewStart"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + "viewUpdate" should "upsert when the row exists and is not completed" in { + val ops = mock[CassandraOperation] + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(uccRow(1)) + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(rowsWith(rows)) + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + val result = callActor(viewRequest("viewUpdate"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + "viewEnd" should "write status=2 and trigger the aggregation" in { + val ops = mock[CassandraOperation] + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + val result = callActor(viewRequest("viewEnd"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + "viewRead" should "return the ucc rows" in { + val ops = mock[CassandraOperation] + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(uccRow(2)) + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(rowsWith(rows)) + val result = callActor(viewRequest("viewRead"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + val out = result.getResult.get("response").asInstanceOf[util.List[util.Map[String, AnyRef]]] + out.size() shouldBe 1 + } + + "viewAssess" should "mark the content complete and trigger aggregation when no events are sent" in { + val ops = mock[CassandraOperation] + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + val result = callActor(viewRequest("viewAssess"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result.getResult.get("ct1") shouldBe "SUCCESS" + } + + "assessmentRead" should "return best score/max score per content" in { + val ops = mock[CassandraOperation] + val attempts = new util.ArrayList[util.Map[String, AnyRef]]() + attempts.add(new util.HashMap[String, AnyRef]() {{ + put("attempt_id", "a1"); put("content_id", "ct1") + put("total_score", java.lang.Double.valueOf(6.0)); put("total_max_score", java.lang.Double.valueOf(10.0)) + }}) + attempts.add(new util.HashMap[String, AnyRef]() {{ + put("attempt_id", "a2"); put("content_id", "ct1") + put("total_score", java.lang.Double.valueOf(8.0)); put("total_max_score", java.lang.Double.valueOf(10.0)) + }}) + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(rowsWith(attempts)) + val result = callActor(viewRequest("assessmentRead"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + val contents = result.getResult.get("contents").asInstanceOf[util.List[util.Map[String, AnyRef]]] + contents.size() shouldBe 1 + contents.get(0).get("identifier") shouldBe "ct1" + // best attempt (8.0) wins + contents.get(0).get("score").asInstanceOf[Double] shouldBe 8.0 + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala new file mode 100644 index 00000000..3b63489b --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala @@ -0,0 +1,70 @@ +package org.sunbird.viewer.actor + +import java.util +import java.util.concurrent.TimeUnit + +import org.apache.pekko.actor.{ActorSystem, Props} +import org.apache.pekko.testkit.TestKit +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.activity.util.{CertificateUtil, HierarchyRelationsUtil} +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.request.{Request, RequestContext} +import org.sunbird.response.Response + +import scala.concurrent.duration.FiniteDuration + +/** + * Unit tests for ViewerAggregatorActor guard branches (deterministic, no hierarchy fixture needed): + * - missing userId/collectionId -> skip, still replies success. + * - no consumption rows -> early return, NO aggregate/enrolment writes. + * The full recursive-rollup happy path depends on a published hierarchy_relations fixture and is + * better exercised as an integration test; these guards pin the cheap-exit correctness. + */ +class ViewerAggregatorActorTest extends AnyFlatSpec with Matchers with MockFactory { + + val system: ActorSystem = ActorSystem.create("viewer-aggregator-test") + + private def emptyRows: Response = { + val r = new Response(); r.put("response", new util.ArrayList[util.Map[String, AnyRef]]()); r + } + + private def callActor(request: Request, props: Props): Response = { + val probe = new TestKit(system) + val actorRef = system.actorOf(props) + actorRef.tell(request, probe.testActor) + probe.expectMsgType[Response](FiniteDuration.apply(15, TimeUnit.SECONDS)) + } + + private def aggRequest(userId: String, collectionId: String): Request = { + val req = new Request + req.setOperation("aggregate") + if (userId != null) req.put("userId", userId) + if (collectionId != null) req.put("collectionId", collectionId) + req.put("contextId", "b1") + req + } + + "aggregate" should "skip and reply success when userId/collectionId are missing" in { + val ops = mock[CassandraOperation] + val hru = mock[HierarchyRelationsUtil] + val cu = mock[CertificateUtil] + // no cassandra / hierarchy / cert interaction expected on the missing-id guard + val result = callActor(aggRequest(null, null), Props(new ViewerAggregatorActor().configure(ops, hru, cu))) + result should not be null + } + + "aggregate" should "early-return with no writes when there is no consumption" in { + val ops = mock[CassandraOperation] + val hru = mock[HierarchyRelationsUtil] + val cu = mock[CertificateUtil] + // not an LP (empty trackablenodes) -> plain rollup path; no consumption -> early return, no writes + (hru.getTrackableNodes(_: String, _: RequestContext)).expects(*, *).returns(List()) + // readConsumption -> empty; must NOT reach batchUpdateWithPutAll / updateRecordV2 / cert + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows) + val result = callActor(aggRequest("u1", "c1"), Props(new ViewerAggregatorActor().configure(ops, hru, cu))) + result should not be null + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala new file mode 100644 index 00000000..e5c249f1 --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala @@ -0,0 +1,93 @@ +package org.sunbird.viewer.util + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class ProgressionPolicySpec extends AnyFlatSpec with Matchers { + + private def node(fields: (String, AnyRef)*): java.util.Map[String, AnyRef] = + fields.toMap.asJava + + private def trackable(id: String, children: java.util.Map[String, AnyRef]*): java.util.Map[String, AnyRef] = + node("identifier" -> id, + "trackable" -> node("enabled" -> "Yes"), + "children" -> children.toList.asJava) + + private def child(primaryCategory: String): java.util.Map[String, AnyRef] = + node("primaryCategory" -> primaryCategory) + + "hasNestedTrackable" should "be true for a trackable collection containing a nested trackable collection" in { + val lp = trackable("do_lp", trackable("CRS-A")) // a trackable collection nested inside a trackable collection + ProgressionPolicy.hasNestedTrackable(lp) shouldBe true + } + + it should "be false for a plain course whose children are non-trackable content" in { + val course = trackable("do_course", node("identifier" -> "c1")) // child is not a trackable collection + ProgressionPolicy.hasNestedTrackable(course) shouldBe false + } + + it should "be false when there are no children at all" in { + ProgressionPolicy.hasNestedTrackable(node("identifier" -> "leaf")) shouldBe false + } + + "isAssessment" should "be true when a Practice Question Set child exists" in { + ProgressionPolicy.isAssessment(trackable("CRS", child("Practice Question Set"))) shouldBe true + } + + it should "be false for a content-only course" in { + ProgressionPolicy.isAssessment(trackable("CRS", child("Explanation Content"))) shouldBe false + } + + // Reference tree: L1[CRS-A] L2[CRS-B,CRS-C] L3[CRS-D,CRS-E] L4[CRS-F] (ancestors nearest-first) + private val anc: Map[String, List[String]] = Map( + "CRS-A" -> List("L1", "do_lp"), + "CRS-B" -> List("L2", "do_lp"), "CRS-C" -> List("L2", "do_lp"), + "CRS-D" -> List("L3", "do_lp"), "CRS-E" -> List("L3", "do_lp"), + "CRS-F" -> List("L4", "do_lp"), + "CRS-X" -> List("grp", "L2", "do_lp")) // nested one level deeper inside L2 + private val ancestorsOf: String => List[String] = anc.getOrElse(_, Nil) + private val order = List("CRS-A", "CRS-B", "CRS-C", "CRS-D", "CRS-E", "CRS-F") + + "levelOf" should "return the level node (the ancestor that is the direct child of the root)" in { + ProgressionPolicy.levelOf("CRS-C", ancestorsOf, "do_lp") shouldBe Some("L2") + } + + it should "pick the direct child of root even when the course is nested deeper (not the nearest ancestor)" in { + ProgressionPolicy.levelOf("CRS-X", ancestorsOf, "do_lp") shouldBe Some("L2") + } + + "coursesOfLevel" should "group the level's courses in trackablenodes order" in { + ProgressionPolicy.coursesOfLevel("L2", order, ancestorsOf, "do_lp") shouldBe List("CRS-B", "CRS-C") + } + + "orderedLevels" should "list levels in first-appearance order" in { + ProgressionPolicy.orderedLevels(order, ancestorsOf, "do_lp") shouldBe List("L1", "L2", "L3", "L4") + } + + "computeOptionalNodes" should "waive a fully-known non-assessment course but never an assessment" in { + val opt = ProgressionPolicy.computeOptionalNodes( + policy = "Adaptive", + courses = List("CRS-B", "CRS-C"), + skillsByCourse = Map("CRS-B" -> Set("s1"), "CRS-C" -> Set("s2")), + assessmentCourses = Set("CRS-C"), + skillsAchieved = Set("s1", "s2")) + opt shouldBe Set("CRS-B") // CRS-C is an assessment -> never optional + } + + it should "waive nothing under Strict" in { + ProgressionPolicy.computeOptionalNodes("Strict", List("CRS-B"), + Map("CRS-B" -> Set("s1")), Set.empty, Set("s1")) shouldBe empty + } + + "computeAchievedSkills" should "return skills whose questions are ALL correct" in { + val skillQs = Map("s1" -> Set("q1", "q2"), "s2" -> Set("q3"), "s3" -> Set("q4", "q5")) + val correct = Set("q1", "q2", "q3", "q4") // s1 all correct, s2 all correct, s3 missing q5 + ProgressionPolicy.computeAchievedSkills(skillQs, correct) shouldBe Set("s1", "s2") + } + + it should "ignore skills that have no tagged questions" in { + ProgressionPolicy.computeAchievedSkills(Map("s0" -> Set.empty[String]), Set("q1")) shouldBe empty + } +} diff --git a/modules/viewer/migrations/viewer-test-keyspace.cql b/modules/viewer/migrations/viewer-test-keyspace.cql new file mode 100644 index 00000000..054f3394 --- /dev/null +++ b/modules/viewer/migrations/viewer-test-keyspace.cql @@ -0,0 +1,154 @@ +/* + * Viewer test keyspace — CREATE-from-scratch (NOT the prod migration). + * + * Purpose: spin up an isolated keyspace with every table the viewer touches, already in the + * generalised (viewer_enabled=true) shape — collectionid/contextid, optional_nodes, user_skills, + * assessment_aggregator on collection_id/context_id. Use it to test the viewer without touching the + * live sunbird_courses. (The prod path is viewer.cql: in-place ALTER ... RENAME on sunbird_courses.) + * + * Usage: + * 1. Pick a keyspace name (default below = sunbird_courses_test). To rename, search/replace + * "sunbird_courses_test" throughout this file. + * 2. Run: cqlsh -f viewer-test-keyspace.cql (or ycqlsh for YugabyteDB) + * 3. Point the service at it: sunbird_course_keyspace=sunbird_courses_test + * (env var or externalresource.properties; read env-first via ProjectUtil.getConfigValue). + * Hierarchy tables are separate (hierarchy_store_keyspace) and are NOT created here. + * + * Target: YugabyteDB (YCQL). The `WITH transactions = {'enabled':'true'}` clauses are required by + * YCQL for tables that carry secondary indexes. On apache Cassandra, drop those WITH clauses. + */ + +CREATE KEYSPACE IF NOT EXISTS sunbird_courses_test + WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; + +/* --- UDT used by assessment_aggregator.question --- */ +CREATE TYPE IF NOT EXISTS sunbird_courses_test.question ( + id text, + assess_ts timestamp, + max_score double, + score double, + type text, + title text, + resvalues frozen>>>, + params frozen>>>, + description text, + duration decimal +); + +/* --- user_content_consumption: per-content view state (viewer ucc). Identity = collection/context. --- */ +CREATE TABLE IF NOT EXISTS sunbird_courses_test.user_content_consumption ( + userid text, + collectionid text, + contextid text, + contentid text, + completedcount int, + completionpercentage float, + datetime timestamp, + last_access_time timestamp, + last_completed_time timestamp, + last_updated_time timestamp, + lastaccesstime text, + lastcompletedtime text, + lastupdatedtime text, + progress int, + progressdetails text, + status int, + viewcount int, + PRIMARY KEY (userid, collectionid, contextid, contentid) +) WITH CLUSTERING ORDER BY (collectionid ASC, contextid ASC, contentid ASC); + +/* --- user_enrolments: per-enrolment progress + per-learner optionality (optional_nodes). --- */ +CREATE TABLE IF NOT EXISTS sunbird_courses_test.user_enrolments ( + userid text, + collectionid text, + contextid text, + active boolean, + addedby text, + certificates list>>, + certstatus int, + completedon timestamp, + completionpercentage int, + contentstatus map, + datetime timestamp, + enrolled_date timestamp, + enrolleddate text, + issued_certificates list>>, + lastcontentaccesstime timestamp, + lastreadcontentid text, + lastreadcontentstatus int, + progress int, + status int, + optional_nodes set, + PRIMARY KEY (userid, collectionid, contextid) +) WITH CLUSTERING ORDER BY (collectionid ASC, contextid ASC) + AND transactions = {'enabled': 'true'}; + +CREATE INDEX IF NOT EXISTS user_enrolments_by_collection ON sunbird_courses_test.user_enrolments (collectionid, userid, contextid) + INCLUDE (status, completionpercentage, enrolled_date, datetime); + +/* --- course_batch: a batch belongs to a collection (root batch or chained rootBatch:courseId child). --- */ +CREATE TABLE IF NOT EXISTS sunbird_courses_test.course_batch ( + collectionid text, + contextid text, + cert_templates map>>, + created_date timestamp, + createdby text, + createddate text, + createdfor list, + description text, + end_date timestamp, + enddate text, + enrollment_enddate timestamp, + enrollmentenddate text, + enrollmenttype text, + mentors list, + name text, + start_date timestamp, + startdate text, + status int, + tandc boolean, + updated_date timestamp, + updateddate text, + PRIMARY KEY (collectionid, contextid) +) WITH CLUSTERING ORDER BY (contextid ASC); + +/* --- user_activity_agg: per-node rollup aggregate (activity_id = node do-id, context_id = "cb:"+contextid). --- */ +CREATE TABLE IF NOT EXISTS sunbird_courses_test.user_activity_agg ( + activity_type text, + activity_id text, + user_id text, + context_id text, + agg map, + agg_details list, + agg_last_updated map, + aggregates map, + PRIMARY KEY ((activity_type, activity_id, user_id), context_id) +) WITH CLUSTERING ORDER BY (context_id ASC); + +/* --- assessment_aggregator: per-attempt scores. Identity generalised to collection_id/context_id. --- */ +CREATE TABLE IF NOT EXISTS sunbird_courses_test.assessment_aggregator ( + collection_id text, + context_id text, + user_id text, + content_id text, + attempt_id text, + created_on timestamp, + grand_total text, + last_attempted_on timestamp, + question list>, + total_max_score double, + total_score double, + updated_on timestamp, + PRIMARY KEY (collection_id, context_id, user_id, content_id, attempt_id) +) WITH CLUSTERING ORDER BY (context_id ASC, user_id ASC, content_id ASC, attempt_id ASC) + AND transactions = {'enabled': 'true'}; + +-- getUserAssessments filters by user_id (not the partition key) -> needs this index. +CREATE INDEX IF NOT EXISTS assessment_aggregator_by_user ON sunbird_courses_test.assessment_aggregator (user_id, collection_id, context_id, content_id, attempt_id) + INCLUDE (total_score, total_max_score, last_attempted_on); + +/* --- user_skills: durable achieved-skill set, credited once at LP completion (design §6). --- */ +CREATE TABLE IF NOT EXISTS sunbird_courses_test.user_skills ( + userid text PRIMARY KEY, + skills set +); diff --git a/modules/viewer/migrations/viewer.cql b/modules/viewer/migrations/viewer.cql new file mode 100644 index 00000000..d0082145 --- /dev/null +++ b/modules/viewer/migrations/viewer.cql @@ -0,0 +1,78 @@ +/* + * Viewer module — generalise the course tables to COLLECTION tables. + * Keyspace: sunbird_courses. Authoritative live schema: + * sunbird-spark-installer/scripts/sunbird-yugabyte-migrations/sunbird-lern/sunbird_courses.cql + * + * DECISION: generalise for collection (not course). Rename the identifying columns to + * collectionid / contextid, and add per-learner optionality. + * + * WHY RENAME (not new table / not backfill): + * - courseid / batchid are PRIMARY-KEY columns; Cassandra/Yugabyte RENAME of PK columns is + * METADATA-ONLY: instant, no data copy, no backfill, no PK restructure (same key, new names). + * - Existing data carries over correctly: old courseid -> collectionid, old batchid -> contextid. + * A course IS a collection, so legacy rows are already right (collectionid = the course do-id, + * contextid = the batch). No row-level migration. + * + * COST (code, not data) — must ship in lockstep: + * - Every reader/writer of courseid/batchid switches to collectionid/contextid: + * ActivityAggregateUtil (createProgressUpdateMap / createContentConsumptionUpdateMap), + * LMS actors, lern-data-pipeline jobs. + * - Secondary index on courseid is dropped + recreated on collectionid (below). + * + * NOTE: verify RENAME support + index handling on the target Yugabyte (YCQL) version before prod. + */ + +/* --- user_content_consumption: generalise identity to collection --- */ +ALTER TABLE sunbird_courses.user_content_consumption RENAME courseid TO collectionid; +ALTER TABLE sunbird_courses.user_content_consumption RENAME batchid TO contextid; + +/* --- user_enrolments: generalise identity + add per-learner optionality --- + * VERIFIED on YCQL (cossdev): a column used in an index CANNOT be renamed + * ("Feature Not Yet Implemented. Can't rename column used in an index"). + * So DROP the index BEFORE renaming, then recreate it on the new name. Order matters. */ +DROP INDEX IF EXISTS sunbird_courses.user_enrolments_by_course; +ALTER TABLE sunbird_courses.user_enrolments RENAME courseid TO collectionid; +ALTER TABLE sunbird_courses.user_enrolments RENAME batchid TO contextid; +ALTER TABLE sunbird_courses.user_enrolments ADD optional_nodes set; -- per-enrolment optional child/leaf ids (null = strict) +-- recreate with the SAME shape as the live index (composite + INCLUDE), just on the new column name +CREATE INDEX IF NOT EXISTS user_enrolments_by_collection ON sunbird_courses.user_enrolments (collectionid, userid, contextid) + INCLUDE (status, completionpercentage, enrolled_date, datetime); -- index build scans the table; run off-peak on large data + +/* --- course_batch: generalise identity (batch belongs to a collection); no index -> rename directly --- */ +ALTER TABLE sunbird_courses.course_batch RENAME courseid TO collectionid; +ALTER TABLE sunbird_courses.course_batch RENAME batchid TO contextid; + +/* --- assessment_aggregator: generalise identity (viewer now OWNS the assessment path) --- + * PK ((course_id, batch_id), user_id, content_id, attempt_id). course_id/batch_id are PARTITION-KEY + * columns -> RENAME is metadata-only (same as the others). The standard by_user index is on user_id + * and is unaffected; IF any index references course_id/batch_id, DROP it first then recreate on the + * new name (see the user_enrolments pattern above). NOTE snake_case names here (matches this table). */ +DROP INDEX IF EXISTS sunbird_courses.assessment_aggregator_by_user; -- only if it references course_id/batch_id; harmless otherwise +ALTER TABLE sunbird_courses.assessment_aggregator RENAME course_id TO collection_id; +ALTER TABLE sunbird_courses.assessment_aggregator RENAME batch_id TO context_id; +-- recreate the by_user index if you dropped it (adjust columns to your live definition): +-- CREATE INDEX IF NOT EXISTS assessment_aggregator_by_user ON sunbird_courses.assessment_aggregator (user_id); + +/* + * Reference — post-rename live tables the viewer uses (do NOT recreate): + * + * course_batch PK (collectionid, contextid) + * user_enrolments PK (userid, collectionid, contextid) -- + optional_nodes + * used by viewer: progress, status, completionpercentage, contentstatus, completedon, lastread* + * user_content_consumption PK (userid, collectionid, contextid, contentid) + * used by viewer: status, progress, progressdetails, completedcount, viewcount, last_*_time + * user_activity_agg -- unchanged; aggregate identified by collectionId (activity_id = collectionId) + * per user + context_id = "cb:"+contextid + * + * assessment_aggregator: PK ((collection_id, context_id), user_id, content_id, attempt_id) -- renamed above + * used by viewer: total_score, total_max_score, grand_total, question, created_on, last_attempted_on + */ + +/* + * LP durable skill store (design §6). Per-user achieved-skill set, credited ONCE at LP completion. + * NOTE: distinct from the legacy `sunbird.user_skills` endorsement table (different keyspace) — no collision. + */ +CREATE TABLE IF NOT EXISTS sunbird_courses.user_skills ( + userid text PRIMARY KEY, + skills set +); diff --git a/modules/viewer/pom.xml b/modules/viewer/pom.xml new file mode 100644 index 00000000..4f3015f5 --- /dev/null +++ b/modules/viewer/pom.xml @@ -0,0 +1,37 @@ + + + + + org.sunbird + lern-service + 1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + viewer + pom + Viewer + Viewer service: granular view APIs + recursive collection tracking + + + + actors + service + + + + + + + com.google.code.play2-maven-plugin + play2-maven-plugin + ${play2.plugin.version} + true + + + + + diff --git a/modules/viewer/service/app/controllers/BaseController.java b/modules/viewer/service/app/controllers/BaseController.java new file mode 100644 index 00000000..28ee00ff --- /dev/null +++ b/modules/viewer/service/app/controllers/BaseController.java @@ -0,0 +1,845 @@ +package controllers; + +import org.apache.pekko.actor.ActorRef; +import org.apache.pekko.actor.ActorSelection; +import org.apache.pekko.pattern.PatternsCS; +import org.apache.pekko.util.Timeout; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import modules.ApplicationStart; +import modules.OnRequestHandler; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.response.Response; +import org.sunbird.response.ResponseParams; +import org.sunbird.operations.lms.ActorOperations; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.common.ProjectUtil; +import org.sunbird.request.HeaderParam; +import org.sunbird.request.RequestContext; +import org.sunbird.response.ResponseCode; +import org.sunbird.keys.SunbirdKey; +import org.sunbird.telemetry.util.TelemetryEvents; +import org.sunbird.telemetry.util.TelemetryWriter; +import play.libs.Json; +import play.mvc.Controller; +import play.mvc.Http; +import play.mvc.Http.Request; +import play.mvc.Result; +import play.mvc.Results; +import util.Attrs; +import util.AuthenticationHelper; + +import java.io.File; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +/** + * This controller we can use for writing some common method. + * + * @author Manzarul + */ +public class BaseController extends Controller { + + private static ObjectMapper objectMapper = new ObjectMapper(); + private static final String version = "v1"; + public static final int PEKKO_WAIT_TIME = 30; + protected Timeout timeout = new Timeout(PEKKO_WAIT_TIME, TimeUnit.SECONDS); + private static final String debugEnabled = "false"; + public static final LoggerUtil logger = new LoggerUtil(BaseController.class); + + private org.sunbird.request.Request initRequest( + org.sunbird.request.Request request, String operation, Http.Request httpRequest) { + request.setOperation(operation); + request.setRequestId(httpRequest.attrs().getOptional(Attrs.REQUEST_ID).orElse(null)); + request.setEnv(getEnvironment()); + request.setRequestContext(getRequestContext(httpRequest, request)); + request.getContext().put(JsonKey.REQUESTED_BY, httpRequest.attrs().getOptional(Attrs.USER_ID).orElse(null)); + request.getRequest().put(JsonKey.REQUESTED_BY, httpRequest.attrs().getOptional(Attrs.USER_ID).orElse(null)); + if (StringUtils.isNotBlank(httpRequest.attrs().getOptional(Attrs.REQUESTED_FOR).orElse(null))) + request.getContext().put(SunbirdKey.REQUESTED_FOR, httpRequest.attrs().get(Attrs.REQUESTED_FOR)); + request.getContext().put(JsonKey.X_AUTH_TOKEN, httpRequest.attrs().getOptional(Attrs.X_AUTH_TOKEN).orElse("")); + request = transformUserId(request); + return request; + } + + private RequestContext getRequestContext(Http.Request httpRequest, org.sunbird.request.Request request) { + RequestContext requestContext = new RequestContext( + JsonKey.SERVICE_NAME, + JsonKey.PRODUCER_NAME, + request.getContext().getOrDefault(JsonKey.ENV, "").toString(), + httpRequest.header(JsonKey.X_DEVICE_ID).orElse(null), + httpRequest.header(JsonKey.X_SESSION_ID).orElse(null), + JsonKey.PID,JsonKey.P_VERSION, null); + requestContext.setActorId(httpRequest.attrs().getOptional(Attrs.ACTOR_ID).orElse(null)); + requestContext.setActorType(httpRequest.attrs().getOptional(Attrs.ACTOR_TYPE).orElse(null)); + requestContext.setRequestId(httpRequest.attrs().getOptional(Attrs.REQUEST_ID).orElse(null)); + return requestContext; + } + + /** + * Helper method for creating and initialising a request for given operation and request body. + * + * @param operation A defined actor operation + * @param requestBodyJson Optional information received in request body (JSON) + * @return Created and initialised Request (@see {@link org.sunbird.request.Request}) + * instance. + */ + protected org.sunbird.request.Request createAndInitRequest( + String operation, JsonNode requestBodyJson, Http.Request httpRequest) { + try { + org.sunbird.request.Request request = + (org.sunbird.request.Request) + mapper.RequestMapper.mapRequest( + requestBodyJson, org.sunbird.request.Request.class); + return initRequest(request, operation, httpRequest); + } catch (Exception e) { + ProjectCommonException.throwServerErrorException(ResponseCode.SERVER_ERROR); + } + return null; + } + + /** + * Helper method for creating and initialising a request for given operation. + * + * @param operation A defined actor operation + * @return Created and initialised Request (@see {@link org.sunbird.request.Request}) + * instance. + */ + protected org.sunbird.request.Request createAndInitRequest( + String operation, Http.Request httpRequest) { + org.sunbird.request.Request request = new org.sunbird.request.Request(); + return initRequest(request, operation, httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + Http.Request httpRequest) { + return handleRequest( + actorRef, + operation, + requestBodyJson, + requestValidatorFn, + null, + null, + null, + true, + httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + java.util.function.Function requestValidatorFn, + Http.Request httpRequest) { + return handleRequest( + actorRef, operation, null, requestValidatorFn, null, null, null, false, httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + String pathId, + String pathVariable, + Http.Request httpRequest) { + return handleRequest( + actorRef, operation, null, null, pathId, pathVariable, null, false, httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + String pathId, + String pathVariable, + boolean isJsonBodyRequired, + Http.Request httpRequest) { + return handleRequest( + actorRef, + operation, + null, + null, + pathId, + pathVariable, + null, + isJsonBodyRequired, + httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + Map headers, + Http.Request httpRequest) { + return handleRequest( + actorRef, + operation, + requestBodyJson, + requestValidatorFn, + null, + null, + headers, + true, + httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + String pathId, + String pathVariable, + Http.Request httpRequest) { + return handleRequest( + actorRef, + operation, + requestBodyJson, + requestValidatorFn, + pathId, + pathVariable, + null, + true, + httpRequest); + } + + protected CompletionStage handleRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + String pathId, + String pathVariable, + Map headers, + boolean isJsonBodyRequired, + Http.Request httpRequest) { + try { + org.sunbird.request.Request request = null; + if (!isJsonBodyRequired) { + request = createAndInitRequest(operation, httpRequest); + } else { + request = createAndInitRequest(operation, requestBodyJson, httpRequest); + } + if (pathId != null) { + request.getRequest().put(pathVariable, pathId); + request.getContext().put(pathVariable, pathId); + } + if (requestValidatorFn != null) requestValidatorFn.apply(request); + if (headers != null) request.getContext().put(JsonKey.HEADER, headers); + + return actorResponseHandler(actorRef, request, timeout, null, httpRequest); + } catch (Exception e) { + logger.error( + "BaseController:handleRequest: Exception occurred with error message = " + e.getMessage(), + e); + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + protected CompletionStage handleSearchRequest( + ActorRef actorRef, + String operation, + JsonNode requestBodyJson, + java.util.function.Function requestValidatorFn, + String pathId, + String pathVariable, + Map headers, + String esObjectType, + Http.Request httpRequest) { + try { + org.sunbird.request.Request request = null; + if (null != requestBodyJson) { + request = createAndInitRequest(operation, requestBodyJson, httpRequest); + } else { + ProjectCommonException.throwClientErrorException(ResponseCode.invalidRequestData, null); + } + if (request != null) { + if (pathId != null) { + request.getRequest().put(pathVariable, pathId); + request.getContext().put(pathVariable, pathId); + } + if (requestValidatorFn != null) requestValidatorFn.apply(request); + if (headers != null) request.getContext().put(JsonKey.HEADER, headers); + if (StringUtils.isNotBlank(esObjectType)) { + List esObjectTypeList = new ArrayList<>(); + esObjectTypeList.add(esObjectType); + ((Map) (request.getRequest().get(JsonKey.FILTERS))) + .put(JsonKey.OBJECT_TYPE, esObjectTypeList); + } + request.getRequest().put(JsonKey.REQUESTED_BY, httpRequest.attrs().getOptional(Attrs.USER_ID).orElse(null)); + } + return actorResponseHandler(actorRef, request, timeout, null, httpRequest); + } catch (Exception e) { + logger.error( + "BaseController:handleRequest: Exception occurred with error message = " + e.getMessage(), + e); + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + /** + * This method will create failure response + * + * @param request Request + * @param code ResponseCode + * @param headerCode ResponseCode + * @return Response + */ + public static Response createFailureResponse( + Request request, ResponseCode code, ResponseCode headerCode) { + + Response response = new Response(); + response.setVer(getApiVersion(request.path())); + response.setId(getApiResponseId(request)); + response.setTs(ProjectUtil.getFormattedDate()); + response.setResponseCode(headerCode); + response.setParams(createResponseParamObj(code, null, request.attrs().getOptional(Attrs.REQUEST_ID).orElse(null))); + return response; + } + + public static ResponseParams createResponseParamObj(ResponseCode code, String customMessage, String requestId) { + ResponseParams params = new ResponseParams(); + if (code.getResponseCode() != 200) { + params.setErr(code.getErrorCode()); + params.setErrmsg( + StringUtils.isNotBlank(customMessage) ? customMessage : code.getErrorMessage()); + } + params.setMsgid(requestId); + params.setStatus(ResponseCode.getHeaderResponseCode(code.getResponseCode()).name()); + return params; + } + + /** + * This method will create data for success response. + * + * @param request play.mvc.Http.Request + * @param response Response + * @return Result + */ + public static Result createSuccessResponse(Request request, Response response) { + if (request != null) { + response.setVer(getApiVersion(request.path())); + } else { + response.setVer(""); + } + + response.setId(getApiResponseId(request)); + response.setTs(ProjectUtil.getFormattedDate()); + ResponseCode code = ResponseCode.getResponse(ResponseCode.success.getErrorCode()); + code.setResponseCode(ResponseCode.OK.getResponseCode()); + response.setParams(createResponseParamObj(code, null, request.attrs().getOptional(Attrs.REQUEST_ID).orElse(null))); + + String value = null; + try { + if (response.getResult() != null) { + String json = new ObjectMapper().writeValueAsString(response.getResult()); + value = getResponseSize(json); + } + } catch (Exception e) { + value = "0.0"; + } + + return Results.ok(Json.toJson(response)) + .withHeader(HeaderParam.X_Response_Length.getName(), value); + } + + /** + * This method will provide api version. + * + * @param request String + * @return String + */ + public static String getApiVersion(String request) { + + return request.split("[/]")[1]; + } + + /** + * This method will handle response in case of exception + * + * @param request play.mvc.Http.Request + * @param exception ProjectCommonException + * @return Response + */ + public static Response createResponseOnException( + Http.Request request, ProjectCommonException exception) { + logger.error( + (org.sunbird.request.RequestContext) null, + exception != null ? exception.getMessage() : "Message is not coming", + exception, + genarateTelemetryInfoForError(request)); + Response response = new Response(); + response.setVer(""); + if (request != null) { + response.setVer(getApiVersion(request.path())); + } + response.setId(getApiResponseId(request)); + response.setTs(ProjectUtil.getFormattedDate()); + if (exception != null) { + response.setResponseCode(ResponseCode.getHeaderResponseCode(exception.getErrorResponseCode())); + ResponseCode code = ResponseCode.getResponse(exception.getCode()); + if (code == null) { + code = ResponseCode.SERVER_ERROR; + } + response.setParams(createResponseParamObj(code, exception.getMessage(), request.attrs().getOptional(Attrs.REQUEST_ID).orElse(null))); + if (response.getParams() != null) { + response.getParams().setStatus(response.getParams().getStatus()); + if (exception.getCode() != null) { + response.getParams().setStatus(exception.getCode()); + } + if (!StringUtils.isBlank(response.getParams().getErrmsg()) + && response.getParams().getErrmsg().contains("{0}")) { + response.getParams().setErrmsg(exception.getMessage()); + } + } + } + return response; + } + + /** + * @param path String + * @param method String + * @param exception ProjectCommonException + * @return Response + */ + public static Response createResponseOnException( + String path, String method, ProjectCommonException exception) { + Response response = new Response(); + response.setVer(getApiVersion(path)); + response.setId(getApiResponseId(path, method)); + response.setTs(ProjectUtil.getFormattedDate()); + response.setResponseCode(ResponseCode.getHeaderResponseCode(exception.getErrorResponseCode())); + ResponseCode code = ResponseCode.getResponse(exception.getCode()); + response.setParams(createResponseParamObj(code, exception.getMessage(), null)); + return response; + } + + /** + * This method will create common response for all controller method + * + * @param response Object + * @param key String + * @param request play.mvc.Http.Request + * @return Result + */ + public Result createCommonResponse(Object response, String key, Http.Request request) { + Response courseResponse = (Response) response; + if (!StringUtils.isBlank(key)) { + Object value = courseResponse.getResult().get(JsonKey.RESPONSE); + courseResponse.getResult().remove(JsonKey.RESPONSE); + courseResponse.getResult().put(key, value); + } + return BaseController.createSuccessResponse(request, courseResponse); + } + + /** + * @param file + * @return + */ + public Result createFileDownloadResponse(File file) { + return Results.ok(file) + .withHeader("Content-Type", "application/x-download") + .withHeader("Content-disposition", "attachment; filename=" + file.getName()); + } + + private void removeFields(Map params, String... properties) { + for (String property : properties) { + params.remove(property); + } + } + + private String generateStackTrace(StackTraceElement[] elements) { + StringBuilder builder = new StringBuilder(""); + for (StackTraceElement element : elements) { + + builder.append(element.toString()); + builder.append("\n"); + } + return ProjectUtil.getFirstNCharacterString(builder.toString(), 100); + } + + private Map generateTelemetryRequestForController( + String eventType, Map params, Map context) { + + Map map = new HashMap<>(); + map.put(JsonKey.TELEMETRY_EVENT_TYPE, eventType); + map.put(JsonKey.CONTEXT, context); + map.put(JsonKey.PARAMS, params); + return map; + } + + /** + * Common exception response handler method. + * + * @param e Exception + * @param request play.mvc.Http.Request + * @return Result + */ + public Result createCommonExceptionResponse(Exception e, Http.Request request) { + Request req = request; + logger.error((org.sunbird.request.RequestContext) null, e.getMessage(), e, genarateTelemetryInfoForError(request)); + ProjectCommonException exception = null; + if (e instanceof ProjectCommonException) { + exception = (ProjectCommonException) e; + } else { + exception = + new ProjectCommonException( + ResponseCode.internalError.getErrorCode(), + ResponseCode.internalError.getErrorMessage(), + ResponseCode.SERVER_ERROR.getResponseCode()); + } + generateExceptionTelemetry(request, exception); + // cleaning request info ... + return Results.status( + exception.getErrorResponseCode(), + Json.toJson(createResponseOnException(req, exception))); + } + + private long calculateApiTimeTaken(Long startTime) { + + Long timeConsumed = null; + if (null != startTime) { + timeConsumed = System.currentTimeMillis() - startTime; + } + return timeConsumed; + } + + /** + * This method will make a call to Pekko actor and return promise. + * + * @param actorRef ActorSelection + * @param request Request + * @param timeout Timeout + * @param responseKey String + * @param httpReq play.mvc.Http.Request + * @return CompletionStage + */ + public CompletionStage actorResponseHandler( + Object actorRef, + org.sunbird.request.Request request, + Timeout timeout, + String responseKey, + Http.Request httpReq) { + + String operation = request.getOperation(); + + // set header to request object , setting actor type and channel headers value + // ... + setContextData(httpReq, request); + setChannelAndActorInfo(httpReq, request); + + Function function = + new Function() { + @Override + public Result apply(Object result) { + if (ActorOperations.HEALTH_CHECK.getValue().equals(request.getOperation())) { + setGlobalHealthFlag(result); + } + + if (result instanceof Response) { + Response response = (Response) result; + return createCommonResponse(response, responseKey, httpReq); + } else if (result instanceof ProjectCommonException) { + return createCommonExceptionResponse((ProjectCommonException) result, httpReq); + } else if (result instanceof File) { + return createFileDownloadResponse((File) result); + } else { + return createCommonExceptionResponse(new Exception(), httpReq); + } + } + }; + + if (actorRef instanceof ActorRef) { + return PatternsCS.ask((ActorRef) actorRef, request, timeout).thenApply(function); + } else { + return PatternsCS.ask((ActorSelection) actorRef, request, timeout).thenApply(function); + } + } + + /** + * This method will provide environment id. + * + * @return int + */ + public int getEnvironment() { + + if (ApplicationStart.env != null) { + return ApplicationStart.env.getValue(); + } + return ProjectUtil.Environment.dev.getValue(); + } + + /** + * Method to get UserId by AuthToken + * + * @param token + * @return String + */ + public String getUserIdByAuthToken(String token) { + + return AuthenticationHelper.verifyUserAccessToken(token); + } + + /** + * Method to get API response Id + * + * @param request play.mvc.Http.Request + * @return String + */ + private static String getApiResponseId(Request request) { + + String val = ""; + if (request != null) { + String path = request.path(); + if (request.method().equalsIgnoreCase(ProjectUtil.Method.GET.name())) { + val = getResponseId(path); + if (StringUtils.isBlank(val)) { + String[] splitedpath = path.split("[/]"); + path = removeLastValue(splitedpath); + val = getResponseId(path); + } + } else { + val = getResponseId(path); + } + if (StringUtils.isBlank(val)) { + val = getResponseId(path); + if (StringUtils.isBlank(val)) { + String[] splitedpath = path.split("[/]"); + path = removeLastValue(splitedpath); + val = getResponseId(path); + } + } + } + return val; + } + + /** + * Method to get API response Id + * + * @param path String + * @param method String + * @return String + */ + private static String getApiResponseId(String path, String method) { + String val = ""; + if (ProjectUtil.Method.GET.name().equalsIgnoreCase(method)) { + val = getResponseId(path); + if (StringUtils.isBlank(val)) { + String[] splitedpath = path.split("[/]"); + String tempPath = removeLastValue(splitedpath); + val = getResponseId(tempPath); + } + } else { + val = getResponseId(path); + } + return val; + } + + /** + * Method to remove last value + * + * @param splited String [] + * @return String + */ + private static String removeLastValue(String splited[]) { + + StringBuilder builder = new StringBuilder(); + if (splited != null && splited.length > 0) { + for (int i = 1; i < splited.length - 1; i++) { + builder.append("/" + splited[i]); + } + } + return builder.toString(); + } + + private static Map genarateTelemetryInfoForError(Http.Request request) { + try{ + Map map = new HashMap<>(); + String reqContext = request.attrs().getOptional(Attrs.CONTEXT).orElse(null); + Map requestInfo = + objectMapper.readValue(reqContext, new TypeReference>() {}); + Map contextInfo = (Map) requestInfo.getOrDefault(JsonKey.CONTEXT, new HashMap()); + Map params = new HashMap<>(); + params.put(JsonKey.ERR_TYPE, JsonKey.API_ACCESS); + + map.put(JsonKey.CONTEXT, contextInfo); + map.put(JsonKey.PARAMS, params); + return map; + } catch (Exception e) { + ProjectCommonException.throwServerErrorException(ResponseCode.SERVER_ERROR); + } + return Collections.emptyMap(); + } + + public void setChannelAndActorInfo( + Http.Request httpReq, org.sunbird.request.Request reqObj) { + + reqObj.getContext().put(JsonKey.CHANNEL, httpReq.attrs().getOptional(Attrs.CHANNEL).orElse(null)); + reqObj.getContext().put(JsonKey.ACTOR_ID, httpReq.attrs().getOptional(Attrs.ACTOR_ID).orElse(null)); + reqObj.getContext().put(JsonKey.ACTOR_TYPE, httpReq.attrs().getOptional(Attrs.ACTOR_TYPE).orElse(null) ); + reqObj.getContext().put(JsonKey.APP_ID, httpReq.attrs().getOptional(Attrs.APP_ID).orElse(null)); + reqObj.getContext().put(JsonKey.DEVICE_ID, httpReq.attrs().getOptional(Attrs.DEVICE_ID).orElse(null)); + reqObj + .getContext() + .put( + JsonKey.SIGNUP_TYPE, + httpReq.attrs().getOptional(Attrs.SIGNUP_TYPE).orElse(null)); // adding signup type in request context + reqObj + .getContext() + .put( + JsonKey.REQUEST_SOURCE, + httpReq.attrs().getOptional(Attrs.REQUEST_SOURCE).orElse(null)); // ADDING Source under params in context + } + + public Map getAllRequestHeaders(Request request) { + Map map = new HashMap<>(); + Map> headers = request.getHeaders().toMap(); + Iterator>> itr = headers.entrySet().iterator(); + while (itr.hasNext()) { + Map.Entry> entry = itr.next(); + map.put(entry.getKey(), entry.getValue().get(0)); + } + return map; + } + + @SuppressWarnings("unchecked") + private void setGlobalHealthFlag(Object result) { + if (result instanceof Response) { + Response response = (Response) result; + if (Boolean.parseBoolean(ProjectUtil.getConfigValue(JsonKey.SUNBIRD_HEALTH_CHECK_ENABLE)) + && ((HashMap) response.getResult().get(JsonKey.RESPONSE)) + .containsKey(JsonKey.Healthy)) { + OnRequestHandler.isServiceHealthy = + (boolean) + ((HashMap) response.getResult().get(JsonKey.RESPONSE)) + .get(JsonKey.Healthy); + } + } else { + OnRequestHandler.isServiceHealthy = false; + } + logger.debug( + "BaseController:setGlobalHealthFlag: isServiceHealthy = " + + OnRequestHandler.isServiceHealthy); + } + + protected String getQueryString(Map queryStringMap) { + return queryStringMap + .entrySet() + .stream() + .map(p -> p.getKey() + "=" + String.join(",", p.getValue())) + .reduce((p1, p2) -> p1 + "&" + p2) + .map(s -> "?" + s) + .orElse(""); + } + + public static String getResponseSize(String response) throws UnsupportedEncodingException { + if (StringUtils.isNotBlank(response)) { + return response.getBytes("UTF-8").length + ""; + } + return "0.0"; + } + + public org.sunbird.request.Request transformUserId( + org.sunbird.request.Request request) { + if (request != null && request.getRequest() != null) { + String id = (String) request.getRequest().get(JsonKey.ID); + request.getRequest().put(JsonKey.ID, ProjectUtil.getLmsUserId(id)); + id = (String) request.getRequest().get(JsonKey.USER_ID); + request.getRequest().put(JsonKey.USER_ID, ProjectUtil.getLmsUserId(id)); + return request; + } + return request; + } + + /** + * Method to get the response id on basis of request path. + * + * @param requestPath + * @return + */ + public static String getResponseId(String requestPath) { + + String path = requestPath; + final String ver = "/" + version; + final String ver2 = "/" + JsonKey.VERSION_2; + path = path.trim(); + StringBuilder builder = new StringBuilder(""); + if (path.startsWith(ver) || path.startsWith(ver2)) { + String requestUrl = (path.split("\\?"))[0]; + if (requestUrl.contains(ver)) { + requestUrl = requestUrl.replaceFirst(ver, "api"); + } else if (requestUrl.contains(ver2)) { + requestUrl = requestUrl.replaceFirst(ver2, "api"); + } + + String[] list = requestUrl.split("/"); + for (String str : list) { + if (str.matches("[A-Za-z]+")) { + builder.append(str).append("."); + } + } + builder.deleteCharAt(builder.length() - 1); + } else { + if ("/health".equalsIgnoreCase(path)) { + builder.append("api.all.health"); + } + } + return builder.toString(); + } + + public void setContextData(Http.Request httpReq, org.sunbird.request.Request reqObj) { + try { + String reqContext = httpReq.attrs().get(Attrs.CONTEXT); + Map requestInfo = + objectMapper.readValue(reqContext, new TypeReference>() {}); + reqObj.setRequestId(httpReq.attrs().getOptional(Attrs.REQUEST_ID).orElse(null)); + reqObj.getContext().putAll((Map) requestInfo.get(JsonKey.CONTEXT)); + reqObj.getContext().putAll((Map) requestInfo.get(JsonKey.ADDITIONAL_INFO)); + } catch (Exception ex) { + ProjectCommonException.throwServerErrorException(ResponseCode.SERVER_ERROR); + } + } + + private void generateExceptionTelemetry(Request request, ProjectCommonException exception) { + try { + String reqContext = request.attrs().get(Attrs.CONTEXT); + Map requestInfo = objectMapper.readValue(reqContext, new TypeReference>() {}); + org.sunbird.request.Request reqForTelemetry = new org.sunbird.request.Request(); + Map params = (Map) requestInfo.getOrDefault(JsonKey.ADDITIONAL_INFO, new HashMap<>()); + params.put(JsonKey.LOG_TYPE, JsonKey.API_ACCESS); + params.put(JsonKey.MESSAGE, ""); + params.put(JsonKey.METHOD, request.method()); + params.put("err", exception.getResponseCode() + ""); + params.put("errtype", exception.getCode()); + long startTime = (Long) params.get(JsonKey.START_TIME); + params.put(JsonKey.DURATION, calculateApiTimeTaken(startTime)); + removeFields(params, JsonKey.START_TIME); + params.put(JsonKey.STATUS, String.valueOf(exception.getResponseCode())); + params.put(JsonKey.LOG_LEVEL, "error"); + params.put(JsonKey.STACKTRACE, generateStackTrace(exception.getStackTrace())); + reqForTelemetry.setRequest( + generateTelemetryRequestForController( + TelemetryEvents.ERROR.getName(), + params, + (Map) requestInfo.get(JsonKey.CONTEXT))); + TelemetryWriter.write(reqForTelemetry); + } catch (Exception ex) { + ex.printStackTrace(); + } + } +} diff --git a/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java b/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java new file mode 100644 index 00000000..87c10496 --- /dev/null +++ b/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java @@ -0,0 +1,55 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.keys.JsonKey; +import org.sunbird.request.Request; +import org.sunbird.response.ResponseCode; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Viewer resync API — recomputes a learner's collection roll-up from user_content_consumption. + * Idempotent (recompute-from-source), so safe to call repeatedly: backfill, drift repair, or a + * recompute after a collection is republished. Mirrors the legacy POST /v1/activity/agg, but targets + * the viewer aggregator (viewer-aggregator-actor, op "aggregate"). Runs on the userId-hashed pool, so + * a resync serialises with any live /v1/view/end for that learner — no race with real-time roll-ups. + * POST /v1/view/agg { request: { userId, collectionId|courseId, contextId|batchId } } + */ +public class ViewAggregateController extends BaseController { + + private final ActorRef viewerAggregatorActor; + + @Inject + public ViewAggregateController(@Named("viewer-aggregator-actor") ActorRef viewerAggregatorActor) { + this.viewerAggregatorActor = viewerAggregatorActor; + } + + public CompletionStage agg(Http.Request httpRequest) { + try { + Request request = createAndInitRequest("aggregate", httpRequest.body().asJson(), httpRequest); + validate(request); + return actorResponseHandler(viewerAggregatorActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + private void validate(Request request) { + String userId = (String) request.get(JsonKey.USER_ID); + Object collectionId = request.get("collectionId") != null ? request.get("collectionId") : request.get(JsonKey.COURSE_ID); + if (userId == null || userId.trim().isEmpty() + || collectionId == null || collectionId.toString().trim().isEmpty()) { + throw new ProjectCommonException( + ResponseCode.mandatoryParamsMissing.getErrorCode(), + "userId and collectionId (or courseId) are mandatory", + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } +} diff --git a/modules/viewer/service/app/controllers/viewer/ViewController.java b/modules/viewer/service/app/controllers/viewer/ViewController.java new file mode 100644 index 00000000..c47b65f4 --- /dev/null +++ b/modules/viewer/service/app/controllers/viewer/ViewController.java @@ -0,0 +1,75 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.request.Request; +import play.mvc.Http; +import play.mvc.Result; + +import play.libs.Json; +import play.mvc.Results; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Granular view lifecycle APIs. Dispatches to ViewConsumptionActor (view-consumption-actor). + * POST /v1/view/start -> viewStart + * POST /v1/view/update -> viewUpdate + * POST /v1/view/end -> viewEnd + */ +public class ViewController extends BaseController { + + private final ActorRef viewConsumptionActor; + + @Inject + public ViewController(@Named("view-consumption-actor") ActorRef viewConsumptionActor) { + this.viewConsumptionActor = viewConsumptionActor; + } + + public CompletionStage viewStart(Http.Request httpRequest) { + return dispatch("viewStart", httpRequest); + } + + public CompletionStage viewUpdate(Http.Request httpRequest) { + return dispatch("viewUpdate", httpRequest); + } + + public CompletionStage viewEnd(Http.Request httpRequest) { + return dispatch("viewEnd", httpRequest); + } + + public CompletionStage viewRead(Http.Request httpRequest) { + return dispatch("viewRead", httpRequest); + } + + public CompletionStage assessmentSubmit(Http.Request httpRequest) { + return dispatch("viewAssess", httpRequest); + } + + public CompletionStage assessmentRead(Http.Request httpRequest) { + return dispatch("assessmentRead", httpRequest); + } + + public Result health(Http.Request httpRequest) { + ObjectNode json = Json.newObject(); + json.put("healthy", true); + return Results.ok(json); + } + + public Result preflight(String all) { + return Results.ok(); + } + + private CompletionStage dispatch(String operation, Http.Request httpRequest) { + try { + Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + return actorResponseHandler(viewConsumptionActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } +} diff --git a/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java b/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java new file mode 100644 index 00000000..2d8dc27e --- /dev/null +++ b/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java @@ -0,0 +1,75 @@ +package controllers.viewer; + +import controllers.BaseController; +import org.apache.pekko.actor.ActorRef; +import org.sunbird.request.Request; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +/** + * Summary APIs. Dispatches to ViewerSummaryActor (viewer-summary-actor). + * POST /v1/summary/read + * GET /v1/summary/list/:userId + * DELETE /v1/summary/delete/:userId + */ +public class ViewSummaryController extends BaseController { + + private final ActorRef viewerSummaryActor; + + @Inject + public ViewSummaryController(@Named("viewer-summary-actor") ActorRef viewerSummaryActor) { + this.viewerSummaryActor = viewerSummaryActor; + } + + public CompletionStage summaryRead(Http.Request httpRequest) { + return dispatchBody("summaryRead", httpRequest); + } + + public CompletionStage summaryList(String userId, Http.Request httpRequest) { + try { + Request request = createAndInitRequest("summaryList", httpRequest); + request.getRequest().put("userId", userId); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + public CompletionStage summaryDownload(String userId, Http.Request httpRequest) { + try { + Request request = createAndInitRequest("summaryDownload", httpRequest); + request.getRequest().put("userId", userId); + String[] fmt = httpRequest.queryString().getOrDefault("format", new String[]{"json"}); + request.getRequest().put("format", fmt.length > 0 ? fmt[0] : "json"); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + public CompletionStage summaryDelete(String userId, Http.Request httpRequest) { + try { + Request request = httpRequest.body().asJson() != null + ? createAndInitRequest("summaryDelete", httpRequest.body().asJson(), httpRequest) + : createAndInitRequest("summaryDelete", httpRequest); + request.getRequest().put("userId", userId); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } + + private CompletionStage dispatchBody(String operation, Http.Request httpRequest) { + try { + Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); + } catch (Exception e) { + return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); + } + } +} diff --git a/modules/viewer/service/app/filters/AccessLogFilter.java b/modules/viewer/service/app/filters/AccessLogFilter.java new file mode 100644 index 00000000..04cd23a4 --- /dev/null +++ b/modules/viewer/service/app/filters/AccessLogFilter.java @@ -0,0 +1,86 @@ +package filters; + +import org.apache.pekko.util.ByteString; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.telemetry.util.TelemetryEvents; +import org.sunbird.telemetry.util.TelemetryWriter; +import play.libs.streams.Accumulator; +import play.mvc.EssentialAction; +import play.mvc.EssentialFilter; +import play.mvc.Result; +import util.Attrs; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; +import java.util.WeakHashMap; +import java.util.concurrent.Executor; + +public class AccessLogFilter extends EssentialFilter { + + private final Executor executor; + private ObjectMapper objectMapper = new ObjectMapper(); + public LoggerUtil logger = new LoggerUtil(this.getClass()); + + @Inject + public AccessLogFilter(Executor executor) { + super(); + this.executor = executor; + } + + @Override + public EssentialAction apply(EssentialAction next) { + return EssentialAction.of( + request -> { + long startTime = System.currentTimeMillis(); + Accumulator accumulator = next.apply(request); + return accumulator.map( + result -> { + long endTime = System.currentTimeMillis(); + long requestTime = endTime - startTime; + try { + org.sunbird.request.Request req = new org.sunbird.request.Request(); + Map params = new WeakHashMap<>(); + params.put(JsonKey.URL, request.uri()); + params.put(JsonKey.METHOD, request.method()); + params.put(JsonKey.LOG_TYPE, JsonKey.API_ACCESS); + params.put(JsonKey.MESSAGE, ""); + params.put(JsonKey.METHOD, request.method()); + params.put(JsonKey.DURATION, requestTime); + params.put(JsonKey.STATUS, result.status()); + params.put(JsonKey.LOG_LEVEL, JsonKey.INFO); + String contextDetails = request.attrs().getOptional(Attrs.CONTEXT).orElse(""); + if(StringUtils.isNotBlank(contextDetails)) { + Map context = + objectMapper.readValue( + contextDetails, new TypeReference>() {}); + req.setRequest( + generateTelemetryRequestForController( + TelemetryEvents.LOG.getName(), + params, + (Map) context.get(JsonKey.CONTEXT))); + TelemetryWriter.write(req); + } + } catch (Exception ex) { + logger.error("AccessLogFilter:apply Exception in writing telemetry", ex); + } + return result; + }, + executor); + }); + } + + private Map generateTelemetryRequestForController( + String eventType, Map params, Map context) { + + Map map = new HashMap<>(); + map.put(JsonKey.TELEMETRY_EVENT_TYPE, eventType); + map.put(JsonKey.CONTEXT, context); + map.put(JsonKey.PARAMS, params); + return map; + } +} diff --git a/modules/viewer/service/app/filters/CustomGzipFilter.java b/modules/viewer/service/app/filters/CustomGzipFilter.java new file mode 100644 index 00000000..f04922dc --- /dev/null +++ b/modules/viewer/service/app/filters/CustomGzipFilter.java @@ -0,0 +1,67 @@ +package filters; + +import org.apache.pekko.stream.Materializer; +import org.apache.http.HttpHeaders; +import org.sunbird.keys.JsonKey; +import org.sunbird.common.ProjectUtil; +import org.sunbird.request.HeaderParam; +import play.filters.gzip.GzipFilter; +import play.filters.gzip.GzipFilterConfig; +import play.mvc.EssentialAction; +import play.mvc.EssentialFilter; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import java.util.function.BiFunction; + +public class CustomGzipFilter extends EssentialFilter { + private static boolean GzipFilterEnabled = + Boolean.parseBoolean(ProjectUtil.getConfigValue(JsonKey.SUNBIRD_GZIP_ENABLE)); + private static final double gzipThreshold = + Double.parseDouble(ProjectUtil.getConfigValue(JsonKey.SUNBIRD_GZIP_SIZE_THRESHOLD)); + private static final String GZIP = "gzip"; + // Size of buffer to use for gzip. + private static final int BUFFER_SIZE = 8192; + // Content length threshold, after which the filter will switch to chunking the result. + private static final int CHUNKED_THRESHOLD = 102400; + + private GzipFilter gzipFilter; + + @Inject + public CustomGzipFilter(Materializer materializer) { + GzipFilterConfig gzipFilterConfig = new GzipFilterConfig(); + gzipFilter = new GzipFilter( + gzipFilterConfig.withBufferSize(BUFFER_SIZE) + .withChunkedThreshold(CHUNKED_THRESHOLD) + .withShouldGzip((BiFunction) + (req, res) -> shouldGzipFunction(req, res)), + materializer + ); + } + + @Override + public EssentialAction apply(EssentialAction essentialAction) { + return gzipFilter.asJava().apply(essentialAction); + } + + // Whether the given request/result should be gzipped or not + private static boolean shouldGzipFunction(Http.RequestHeader requestHeader, Result responseHeader) { + double responseSize = 0.0; + boolean responseLengthKeyExist = responseHeader.headers().containsKey(HeaderParam.X_Response_Length.getName()); + if (responseLengthKeyExist) { + if (responseHeader.headers().get(HeaderParam.X_Response_Length.getName()) != null) { + String strValue = responseHeader.headers().get(HeaderParam.X_Response_Length.getName()); + responseSize = Double.parseDouble(strValue); + } + } + if (GzipFilterEnabled && (requestHeader.header(HttpHeaders.ACCEPT_ENCODING) != null)) { + if (requestHeader.header(HttpHeaders.ACCEPT_ENCODING).toString().toLowerCase().contains(GZIP)) { + if (responseSize >= gzipThreshold) { + return true; + } + } + } + return false; + } +} diff --git a/modules/viewer/service/app/filters/LoggingFilter.java b/modules/viewer/service/app/filters/LoggingFilter.java new file mode 100644 index 00000000..f45b9c88 --- /dev/null +++ b/modules/viewer/service/app/filters/LoggingFilter.java @@ -0,0 +1,42 @@ +package filters; + +import org.apache.pekko.stream.Materializer; +import play.Logger; +import play.mvc.Filter; +import play.mvc.Http; +import play.mvc.Result; + +import javax.inject.Inject; +import java.util.concurrent.CompletionStage; +import java.util.function.Function; + +public class LoggingFilter extends Filter { + + @Inject + public LoggingFilter(Materializer mat) { + super(mat); + } + + @Override + public CompletionStage apply( + Function> nextFilter, + Http.RequestHeader requestHeader) { + long startTime = System.currentTimeMillis(); + return nextFilter + .apply(requestHeader) + .thenApply( + result -> { + long endTime = System.currentTimeMillis(); + long requestTime = endTime - startTime; + + Logger.info( + "{} {} took {}ms and returned {}", + requestHeader.method(), + requestHeader.uri(), + requestTime, + result.status()); + + return result.withHeader("Request-Time", "" + requestTime); + }); + } +} \ No newline at end of file diff --git a/modules/viewer/service/app/filters/ResponseFilter.scala b/modules/viewer/service/app/filters/ResponseFilter.scala new file mode 100644 index 00000000..60469bad --- /dev/null +++ b/modules/viewer/service/app/filters/ResponseFilter.scala @@ -0,0 +1,40 @@ +package filters + +import org.apache.pekko.stream.Materializer +import org.apache.pekko.util.ByteString +import org.apache.commons.lang3.StringUtils +import org.sunbird.keys.JsonKey +import org.sunbird.keys.JsonKey.{CLOUD_STORAGE_CNAME_URL, CLOUD_STORE_BASE_PATH, CONTENT_CLOUD_STORAGE_CONTAINER} +import org.sunbird.common.ProjectUtil.getConfigValue +import play.api.Logging +import play.api.http.HttpEntity.Strict +import play.api.mvc.{Filter, RequestHeader, Result} + +import javax.inject.Inject +import scala.concurrent.{ExecutionContext, Future} + +class ResponseFilter @Inject()(implicit val mat: Materializer, ec: ExecutionContext) extends Filter with Logging { + + override def apply(nextFilter: (RequestHeader) => Future[Result])(rh: RequestHeader) = + nextFilter(rh) flatMap { result => + if (null != result.body && !result.body.isKnownEmpty){ + val contentType = result.body.contentType + val updatedBody = result.body.consumeData.map { x => + val y = x.utf8String.replaceAll(getConfigValue(JsonKey.CLOUD_STORE_BASE_PATH_PLACEHOLDER), getBaseUrl + "/" + getConfigValue(CONTENT_CLOUD_STORAGE_CONTAINER)) + logger.info("updated body: " + y) + y + } + updatedBody map { x => + result.copy(body = Strict(ByteString(x), contentType)) + } + } else { + Future(result) + } + } + + def getBaseUrl: String = { + var baseUrl = getConfigValue(CLOUD_STORAGE_CNAME_URL) + if (StringUtils.isEmpty(baseUrl)) baseUrl = getConfigValue(CLOUD_STORE_BASE_PATH) + baseUrl + } +} \ No newline at end of file diff --git a/modules/viewer/service/app/mapper/RequestMapper.java b/modules/viewer/service/app/mapper/RequestMapper.java new file mode 100644 index 00000000..7e391865 --- /dev/null +++ b/modules/viewer/service/app/mapper/RequestMapper.java @@ -0,0 +1,225 @@ +/** */ +package mapper; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.common.ProjectUtil; +import org.sunbird.request.Request; +import org.sunbird.response.ResponseCode; +import play.libs.Json; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * This class will map the requested json data into custom class. + * + * @author Manzarul + */ +public class RequestMapper { + public static LoggerUtil logger = new LoggerUtil(RequestMapper.class); + + /** + * Method to map request + * + * @param requestData JsonNode + * @param obj Class + * @exception RuntimeException + * @return + */ + public static Object mapRequest(JsonNode requestData, Class obj) throws Exception { + // First convert the JsonNode through our Scala collection conversion + Map convertedMap = mapRequest(requestData); + + // Now convert the cleaned map to the target object type + ObjectMapper mapper = new ObjectMapper(); + Object result = mapper.convertValue(convertedMap, obj); + + // For Request objects, ensure the internal request map also uses converted collections + if (result instanceof Request) { + Request requestObj = (Request) result; + Map internalRequest = requestObj.getRequest(); + if (internalRequest != null) { + Map convertedInternalRequest = new HashMap<>(); + for (Map.Entry entry : internalRequest.entrySet()) { + convertedInternalRequest.put(entry.getKey(), convertScalaCollections(entry.getValue())); + } + requestObj.setRequest(convertedInternalRequest); + } + } + + return result; + } + + /** + * Method to map request with Scala collection conversion + * + * @param requestData JsonNode + * @exception RuntimeException + * @return Map + */ + public static Map mapRequest(JsonNode requestData) throws Exception { + if (requestData == null) { + return new HashMap<>(); + } + + ObjectMapper mapper = new ObjectMapper(); + Map map = mapper.convertValue(requestData, new TypeReference>() {}); + + // Convert any Scala collections (Maps, Lists, etc.) to Java collections + Map convertedMap = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + Object convertedValue = convertScalaCollections(entry.getValue()); + convertedMap.put(entry.getKey(), convertedValue); + } + + return convertedMap; + } + + /** + * Helper method to convert Scala Map to Java Map recursively, including List conversions + * @param obj The Scala Map to convert + * @return Java Map + */ + private static Object convertToJavaMap(Object obj) { + if (obj instanceof Map && !(obj instanceof java.util.Map)) { + try { + // Convert Scala Map to Java Map + scala.collection.Map scalaMap = (scala.collection.Map) obj; + Map javaMap = new LinkedHashMap<>(); + + // Iterate through Scala map and convert each entry + scala.collection.Iterator iterator = scalaMap.iterator(); + while (iterator.hasNext()) { + scala.Tuple2 entry = (scala.Tuple2) iterator.next(); + Object key = entry._1(); + Object value = entry._2(); + + // Recursively convert nested maps and handle Scala collections + Object convertedValue = convertScalaCollections(value); + javaMap.put(key, convertedValue); + } + + return javaMap; + } catch (Exception e) { + logger.debug("Failed to convert Scala Map to Java Map: " + e.getMessage() + + ". Object type: " + obj.getClass().getName() + + ". Returning original object. Exception: " + e.toString()); + return obj; + } + } else { + return obj; + } + } + + /** + * Convert Scala collections to Java collections recursively + */ + private static Object convertScalaCollections(Object obj) { + if (obj == null) { + return null; + } + + // Handle Scala Lists (including :: cons lists) + if (obj instanceof scala.collection.Seq && !(obj instanceof java.util.List)) { + try { + scala.collection.Seq scalaSeq = (scala.collection.Seq) obj; + List javaList = new ArrayList<>(); + + scala.collection.Iterator iterator = scalaSeq.iterator(); + while (iterator.hasNext()) { + Object element = iterator.next(); + // Recursively convert nested collections + Object convertedElement = convertScalaCollections(element); + javaList.add(convertedElement); + } + + return javaList; + } catch (Exception e) { + logger.debug("Failed to convert Scala Seq to Java List: " + e.getMessage() + + ". Object type: " + obj.getClass().getName() + + ". Returning original object. Exception: " + e.toString()); + return obj; + } + } + + // Handle Scala Maps + if (obj instanceof scala.collection.Map && !(obj instanceof java.util.Map)) { + return convertToJavaMap(obj); + } + + // Handle Java Lists (process recursively for nested collections) + if (obj instanceof java.util.List) { + List javaList = (List) obj; + List convertedList = new ArrayList<>(); + for (Object element : javaList) { + convertedList.add(convertScalaCollections(element)); + } + return convertedList; + } + + // Handle Java Maps (process recursively for nested collections) + if (obj instanceof java.util.Map) { + Map javaMap = (Map) obj; + Map convertedMap = new LinkedHashMap<>(); + for (Map.Entry entry : javaMap.entrySet()) { + Object convertedValue = convertScalaCollections(entry.getValue()); + convertedMap.put(entry.getKey(), convertedValue); + } + return convertedMap; + } + + // Return other objects as-is + return obj; + } + + /** + * Helper method to convert Scala collections to Java collections + * @param value The value to potentially convert + * @param fieldName The field name for logging + * @return Java collection or original value + */ + private static Object convertScalaCollectionToJava(Object value, String fieldName) { + if (value == null) { + return value; + } + + String className = value.getClass().getName(); + + // Handle all Scala collection types + if (className.startsWith("scala.collection")) { + // Check if it's a Scala List (including :: cons lists) + if (className.contains("List") || className.contains("$colon$colon") || + className.contains("immutable.Nil") || value instanceof Iterable) { + List javaList = new ArrayList<>(); + + try { + // Handle Scala collections that are Iterable + if (value instanceof Iterable) { + for (Object item : (Iterable) value) { + // Recursively convert nested structures + if (item instanceof scala.collection.Map && !(item instanceof java.util.Map)) { + item = convertToJavaMap(item); + } else { + item = convertScalaCollectionToJava(item, fieldName + "[element]"); + } + javaList.add(item); + } + } + + return javaList; + + } catch (Exception e) { + logger.error("Failed to convert Scala collection for field " + fieldName + ": " + e.getMessage(), e); + return value; // Return original value if conversion fails + } + } + } + + return value; + } +} diff --git a/modules/viewer/service/app/modules/ActorStartModule.java b/modules/viewer/service/app/modules/ActorStartModule.java new file mode 100644 index 00000000..0446791a --- /dev/null +++ b/modules/viewer/service/app/modules/ActorStartModule.java @@ -0,0 +1,42 @@ +package modules; + +import org.apache.pekko.routing.ConsistentHashingPool; +import org.apache.pekko.routing.ConsistentHashingRouter.ConsistentHashMapper; +import org.apache.pekko.routing.FromConfig; +import org.apache.pekko.routing.RouterConfig; +import com.google.inject.AbstractModule; +import play.libs.pekko.PekkoGuiceSupport; +import util.ACTOR_NAMES; + +public class ActorStartModule extends AbstractModule implements PekkoGuiceSupport { + + @Override + protected void configure() { + System.out.println("binding actors for dependency injection"); + final RouterConfig config = new FromConfig(); + + // viewer-aggregator: PROGRAMMATIC ConsistentHashingPool keyed on userId — serialises one user's + // roll-ups. A config-only consistent-hashing router has no hash key for plain Request messages + // and would send them to deadLetters. + final ConsistentHashMapper userIdHashMapper = + message -> { + if (message instanceof org.sunbird.request.Request) { + Object uid = ((org.sunbird.request.Request) message).get("userId"); + return uid != null ? uid : ""; + } + return ""; + }; + + for (ACTOR_NAMES actor : ACTOR_NAMES.values()) { + if (ACTOR_NAMES.VIEWER_AGGREGATOR_ACTOR.equals(actor)) { + bindActor( + actor.getActorClass(), + actor.getActorName(), + props -> props.withRouter(new ConsistentHashingPool(8).withHashMapper(userIdHashMapper))); + } else { + bindActor(actor.getActorClass(), actor.getActorName(), props -> props.withRouter(config)); + } + } + System.out.println("binding completed"); + } +} diff --git a/modules/viewer/service/app/modules/ApplicationStart.java b/modules/viewer/service/app/modules/ApplicationStart.java new file mode 100644 index 00000000..5e998ee1 --- /dev/null +++ b/modules/viewer/service/app/modules/ApplicationStart.java @@ -0,0 +1,83 @@ +package modules; + +import org.sunbird.auth.verifier.KeyManager; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.common.ProjectUtil; +import org.sunbird.learner.util.ContentSearchMock; +import org.sunbird.learner.util.SchedulerManager; +import org.sunbird.learner.util.Util; +import play.api.Environment; +import play.api.inject.ApplicationLifecycle; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.concurrent.CompletableFuture; + +/** + * This class will be called after on application startup. only one instance of this class will be + * created. StartModule class has responsibility to eager load this class. + * + * @author Jaikumar Soundara Rajan + */ +@Singleton +public class ApplicationStart { + + public static ProjectUtil.Environment env; + public static String ssoPublicKey = ""; + public LoggerUtil logger = new LoggerUtil(this.getClass()); + + /** + * All one time initialization which required during server startup will fall here. + * + * @param lifecycle ApplicationLifecycle + * @param environment Environment + */ + @Inject + public ApplicationStart(ApplicationLifecycle lifecycle, Environment environment) { + System.out.println("ApplicationStart:ApplicationStart: Start"); + setEnvironment(environment); + ssoPublicKey = System.getenv(JsonKey.SSO_PUBLIC_KEY); + logger.info("Server started.. with environment: " + env.name()); + if (Boolean.parseBoolean(ProjectUtil.getConfigValue(JsonKey.CONTENT_SERVICE_MOCK_ENABLED))) { + mockServiceSetup(); + } + checkCassandraConnections(); + SchedulerManager.schedule(); + lifecycle.addStopHook( + () -> { + return CompletableFuture.completedFuture(null); + }); + System.out.println("keymanger.init():starts"); + KeyManager.init(); + System.out.println("ApplicationStart:ApplicationStart: End"); + } + + public static void mockServiceSetup() { + LoggerUtil logger = new LoggerUtil(ApplicationStart.class); + try { + ContentSearchMock.setup(); + } catch (Exception e) { + logger.info((org.sunbird.request.RequestContext) null,"Error setting up ContentSearchMock:"+e); + } + } + + private void checkCassandraConnections() { + Util.checkCassandraDbConnections(); + } + + /** + * This method will identify the environment and update with enum. + * + * @return Environment + */ + public ProjectUtil.Environment setEnvironment(Environment environment) { + if (environment.asJava().isDev()) { + return env = ProjectUtil.Environment.dev; + } else if (environment.asJava().isTest()) { + return env = ProjectUtil.Environment.qa; + } else { + return env = ProjectUtil.Environment.prod; + } + } +} diff --git a/modules/viewer/service/app/modules/ErrorHandler.java b/modules/viewer/service/app/modules/ErrorHandler.java new file mode 100644 index 00000000..a275fc61 --- /dev/null +++ b/modules/viewer/service/app/modules/ErrorHandler.java @@ -0,0 +1,61 @@ +package modules; + +import com.typesafe.config.Config; +import controllers.BaseController; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.response.Response; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.response.ResponseCode; +import play.Environment; +import play.api.OptionalSourceMapper; +import play.api.routing.Router; +import play.http.DefaultHttpErrorHandler; +import play.libs.Json; +import play.mvc.Http; +import play.mvc.Result; +import play.mvc.Results; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +@Singleton +public class ErrorHandler extends DefaultHttpErrorHandler { + public LoggerUtil logger = new LoggerUtil(this.getClass()); + + @Inject + public ErrorHandler( + Config config, + Environment environment, + OptionalSourceMapper sourceMapper, + Provider routes) { + super(config, environment, sourceMapper, routes); + } + + @Override + public CompletionStage onServerError(Http.RequestHeader request, Throwable t) { + logger.error("Global: onError called for path = " + request.path(), t); + Response response = null; + ProjectCommonException commonException = null; + if (t instanceof ProjectCommonException) { + commonException = (ProjectCommonException) t; + } else if (t instanceof org.apache.pekko.pattern.AskTimeoutException) { + commonException = + new ProjectCommonException( + ResponseCode.actorConnectionError.getErrorCode(), + ResponseCode.actorConnectionError.getErrorMessage(), + ResponseCode.SERVER_ERROR.getResponseCode()); + } else { + commonException = + new ProjectCommonException( + ResponseCode.internalError.getErrorCode(), + ResponseCode.internalError.getErrorMessage(), + ResponseCode.SERVER_ERROR.getResponseCode()); + } + response = + BaseController.createResponseOnException(request.path(), request.method(), commonException); + return CompletableFuture.completedFuture(Results.internalServerError(Json.toJson(response))); + } +} diff --git a/modules/viewer/service/app/modules/OnRequestHandler.java b/modules/viewer/service/app/modules/OnRequestHandler.java new file mode 100644 index 00000000..d77b6acf --- /dev/null +++ b/modules/viewer/service/app/modules/OnRequestHandler.java @@ -0,0 +1,283 @@ +package modules; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.typesafe.config.ConfigFactory; +import controllers.BaseController; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.auth.verifier.AccessTokenValidator; +import org.sunbird.cache.platform.Platform; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.response.Response; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.common.ProjectUtil; +import org.sunbird.request.HeaderParam; +import org.sunbird.response.ResponseCode; +import org.sunbird.utils.JsonUtil; +import play.http.ActionCreator; +import play.libs.Json; +import play.mvc.Action; +import play.mvc.Http; +import play.mvc.Result; +import play.mvc.Results; +import util.Attrs; +import util.RequestInterceptor; + +import java.lang.reflect.Method; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.WeakHashMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.stream.Collectors; + +public class OnRequestHandler implements ActionCreator { + + private ObjectMapper mapper = new ObjectMapper(); + public static boolean isServiceHealthy = true; + private final List USER_UNAUTH_STATES = + Arrays.asList(JsonKey.UNAUTHORIZED, JsonKey.ANONYMOUS); + public LoggerUtil logger = new LoggerUtil(this.getClass()); + private static final List clientAppHeaderKeys = Platform.getStringList("request_headers_logging", Arrays.asList("x-app-id", "x-device-id", "x-channel-id")); + + + @Override + public Action createAction(Http.Request request, Method actionMethod) { + Optional optionalMessageId = request.header(JsonKey.MESSAGE_ID); + String messageId; + if (optionalMessageId.isPresent()) { + messageId = optionalMessageId.get(); + } else { + UUID uuid = UUID.randomUUID(); + messageId = uuid.toString(); + } + return new Action.Simple() { + @Override + public CompletionStage call(Http.Request request) { + CompletionStage result = checkForServiceHealth(request); + String message = null; + if (result != null) { + return result; + } + if (ConfigFactory.load().getBoolean(JsonKey.AUTH_ENABLED)) { + message = RequestInterceptor.verifyRequestData(request); + } else { + message = JsonKey.ANONYMOUS; + } + // Setting Actual userId (requestedBy) and managed userId (requestedFor) placeholders in flash memory to null before processing. + // Unauthorized, Anonymous, UserID + Optional forAuth = request.header(HeaderParam.X_Authenticated_For.getName()); + String childId = null; + String loggingHeaders = getLoggingHeaders(request); + request = request.addAttr(Attrs.X_LOGGING_HEADERS, loggingHeaders); + if (StringUtils.isNotBlank(message) && forAuth.isPresent() && StringUtils.isNotBlank(forAuth.orElse(""))) { + String requestedForId = getRequestedForId(request); + childId = AccessTokenValidator.verifyManagedUserToken(forAuth.get(), message, requestedForId, loggingHeaders); + if (StringUtils.isNotBlank(childId) && !USER_UNAUTH_STATES.contains(childId)) { + request = request.addAttr(Attrs.REQUESTED_FOR, childId); + } + + } + // call method to set all the required params for the telemetry event(log)... + request = intializeRequestInfo(request, message, messageId); + request = request.addAttr(Attrs.X_AUTH_TOKEN, request.header(HeaderParam.X_Authenticated_User_Token.getName()).orElse("")); + if ((!USER_UNAUTH_STATES.contains(message)) && (childId==null || !USER_UNAUTH_STATES.contains(childId))) { + request = request.addAttr(Attrs.USER_ID, message); + request = request.addAttr(Attrs.IS_AUTH_REQ, "false"); + for (String uri : RequestInterceptor.restrictedUriList) { + if (request.path().contains(uri)) { + request = request.addAttr(Attrs.IS_AUTH_REQ, "true"); + break; + } + } + result = delegate.call(request); + } else if (JsonKey.UNAUTHORIZED.equals(message) || (childId != null && JsonKey.UNAUTHORIZED.equals(childId))) { + String errorCode = JsonKey.UNAUTHORIZED.equals(message) ? message : childId; + result = onDataValidationError(request, errorCode, ResponseCode.UNAUTHORIZED.getResponseCode()); + } else { + result = delegate.call(request); + } + return result.thenApply(res -> res.withHeader("Access-Control-Allow-Origin", "*")); + } + }; + } + + private String getRequestedForId(Http.Request request) { + String requestedForUserID = null; + JsonNode jsonBody = request.body().asJson(); + if(null != jsonBody && jsonBody.has(JsonKey.REQUEST) && jsonBody.get(JsonKey.REQUEST).has(JsonKey.USER_ID)) { + requestedForUserID = jsonBody.get(JsonKey.REQUEST).get(JsonKey.USER_ID).asText(); + } else { // for read-api + String uuidSegment = null; + Path path = Paths.get(request.uri()); + if (request.queryString().isEmpty()) { + uuidSegment = path.getFileName().toString(); + } else { + String[] queryPath = path.getFileName().toString().split("\\?"); + uuidSegment = queryPath[0]; + } + try { + requestedForUserID = UUID.fromString(uuidSegment).toString(); + } catch (IllegalArgumentException iae) { + logger.info("Perhaps this is another API, like search that doesn't carry user id."); + } + } + return requestedForUserID; + } + + /** + * This method will do request data validation for GET method only. As a GET request user must + * send some key in header. + * + * @param request Request + * @param errorMessage String + * @return Promise + */ + public CompletionStage onDataValidationError( + Http.Request request, String errorMessage, int responseCode) { + logger.info("Data error found--" + errorMessage); + ResponseCode code = ResponseCode.getResponse(errorMessage); + ResponseCode headerCode = ResponseCode.CLIENT_ERROR; + Response resp = BaseController.createFailureResponse(request, code, headerCode); + return CompletableFuture.completedFuture(Results.status(responseCode, Json.toJson(resp))); + } + + private Http.Request intializeRequestInfo(Http.Request request, String userId, String requestId) { + try { + String actionMethod = request.method(); + String url = request.uri(); + String methodName = actionMethod; + long startTime = System.currentTimeMillis(); + String signType = ""; + String source = ""; + if (request.body() != null && request.body().asJson() != null) { + JsonNode requestNode = + request.body().asJson().get("params"); // extracting signup type from request + if (requestNode != null && requestNode.get(JsonKey.SIGNUP_TYPE) != null) { + signType = requestNode.get(JsonKey.SIGNUP_TYPE).asText(); + } + if (requestNode != null && requestNode.get(JsonKey.REQUEST_SOURCE) != null) { + source = requestNode.get(JsonKey.REQUEST_SOURCE).asText(); + } + } + Map reqContext = new WeakHashMap<>(); + request = request.addAttr(Attrs.SIGNUP_TYPE, signType); + reqContext.put(JsonKey.SIGNUP_TYPE, signType); + request = request.addAttr(Attrs.REQUEST_SOURCE, source); + reqContext.put(JsonKey.REQUEST_SOURCE, source); + + // set env and channel to the + Optional optionalChannel = request.header(HeaderParam.CHANNEL_ID.getName()); + String channel; + if (optionalChannel.isPresent()) { + channel = optionalChannel.get(); + } else { + String sunbirdDefaultChannel = ProjectUtil.getConfigValue(JsonKey.SUNBIRD_DEFAULT_CHANNEL); + channel = + (StringUtils.isNotEmpty(sunbirdDefaultChannel)) + ? sunbirdDefaultChannel + : JsonKey.DEFAULT_ROOT_ORG_ID; + } + reqContext.put(JsonKey.CHANNEL, channel); + request = request.addAttr(Attrs.CHANNEL, channel); + reqContext.put(JsonKey.ENV, getEnv(request)); + reqContext.put(JsonKey.REQUEST_ID, requestId); + Optional optionalAppId = request.header(HeaderParam.X_APP_ID.getName()); + // check if in request header X-app-id is coming then that need to + // be pass in search telemetry. + if (optionalAppId.isPresent()) { + request = request.addAttr(Attrs.APP_ID, optionalAppId.get()); + reqContext.put(JsonKey.APP_ID, optionalAppId.get()); + } + // checking device id in headers + Optional optionalDeviceId = request.header(HeaderParam.X_Device_ID.getName()); + if (optionalDeviceId.isPresent()) { + request = request.addAttr(Attrs.DEVICE_ID, optionalDeviceId.get()); + reqContext.put(JsonKey.DEVICE_ID, optionalDeviceId.get()); + } + if (!USER_UNAUTH_STATES.contains(userId)) { + reqContext.put(JsonKey.ACTOR_ID, userId); + reqContext.put(JsonKey.ACTOR_TYPE, StringUtils.capitalize(JsonKey.USER)); + request = request.addAttr(Attrs.ACTOR_ID, userId); + request = request.addAttr(Attrs.ACTOR_TYPE, JsonKey.USER); + } else { + Optional optionalConsumerId = request.header(HeaderParam.X_Consumer_ID.getName()); + String consumerId; + if (optionalConsumerId.isPresent()) { + consumerId = optionalConsumerId.get(); + } else { + consumerId = JsonKey.DEFAULT_CONSUMER_ID; + } + reqContext.put(JsonKey.ACTOR_ID, consumerId); + reqContext.put(JsonKey.ACTOR_TYPE, StringUtils.capitalize(JsonKey.CONSUMER)); + request = request.addAttr(Attrs.ACTOR_ID, consumerId); + request = request.addAttr(Attrs.ACTOR_TYPE, JsonKey.CONSUMER); + } + Map map = new WeakHashMap<>(); + map.put(JsonKey.CONTEXT, reqContext); + Map additionalInfo = new WeakHashMap<>(); + additionalInfo.put(JsonKey.URL, url); + additionalInfo.put(JsonKey.METHOD, methodName); + additionalInfo.put(JsonKey.START_TIME, startTime); + + // additional info contains info other than context info ... + map.put(JsonKey.ADDITIONAL_INFO, additionalInfo); + if (StringUtils.isBlank(requestId)) { + requestId = JsonKey.DEFAULT_CONSUMER_ID; + } + request = request.addAttr(Attrs.REQUEST_ID, requestId); + request = request.addAttr(Attrs.CONTEXT, mapper.writeValueAsString(map)); + } catch (Exception e) { + ProjectCommonException.throwServerErrorException(ResponseCode.SERVER_ERROR, e.getMessage()); + } + return request; + } + + private String getEnv(Http.Request request) { + + String uri = request.uri(); + String env; + if (uri.startsWith("/v1/page")) { + env = JsonKey.PAGE; + } else if (uri.startsWith("/v1/course/batch")) { + env = JsonKey.BATCH; + } else if (uri.startsWith("/v1/dashboard")) { + env = JsonKey.DASHBOARD; + } else if (uri.startsWith("/v1/content")) { + env = JsonKey.BATCH; + } else { + env = "miscellaneous"; + } + return env; + } + + public CompletionStage checkForServiceHealth(Http.Request request) { + if (Boolean.parseBoolean((ProjectUtil.getConfigValue(JsonKey.SUNBIRD_HEALTH_CHECK_ENABLE))) + && !request.path().endsWith(JsonKey.HEALTH)) { + if (!isServiceHealthy) { + ResponseCode headerCode = ResponseCode.SERVICE_UNAVAILABLE; + Response resp = BaseController.createFailureResponse(request, headerCode, headerCode); + return CompletableFuture.completedFuture( + Results.status(ResponseCode.SERVICE_UNAVAILABLE.getResponseCode(), Json.toJson(resp))); + } + } + return null; + } + + // TODO: same method created in BaseController also. We should move it to a common place. + protected String getLoggingHeaders(Http.Request httpRequest) { + try { + Map> headers = httpRequest.getHeaders().toMap(); + Map> filteredHeaders = headers.entrySet().stream().filter(e -> clientAppHeaderKeys.contains(e.getKey().toLowerCase())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return JsonUtil.serialize(filteredHeaders); + } catch (Exception e) { + return "Exception in serializing headers= " + e.getMessage(); + } + } +} diff --git a/modules/viewer/service/app/modules/StartModule.java b/modules/viewer/service/app/modules/StartModule.java new file mode 100644 index 00000000..3f9ba7d8 --- /dev/null +++ b/modules/viewer/service/app/modules/StartModule.java @@ -0,0 +1,29 @@ +package modules; + +import com.google.inject.AbstractModule; +import org.sunbird.cache.util.RedisCacheUtil; +import org.sunbird.common.ProjectUtil; + +/** + * This class is responsible for creating instance of + * ApplicationStart at server startup time. + * + * @author Jaikumar Soundara Rajan + */ +public class StartModule extends AbstractModule { + @Override + protected void configure() { + System.out.println("StartModule:configure: Start"); + try { + bind(ApplicationStart.class).asEagerSingleton(); + if (Boolean.parseBoolean(ProjectUtil.getConfigValue("redis.enabled"))) { + bind(RedisCacheUtil.class).asEagerSingleton(); + } + } catch (Exception | Error e) { + e.printStackTrace(); + throw e; + } + System.out.println("StartModule:configure: End"); + + } +} diff --git a/modules/viewer/service/app/util/ACTOR_NAMES.java b/modules/viewer/service/app/util/ACTOR_NAMES.java new file mode 100644 index 00000000..c9bdc520 --- /dev/null +++ b/modules/viewer/service/app/util/ACTOR_NAMES.java @@ -0,0 +1,27 @@ +package util; + +import org.sunbird.viewer.actor.ViewConsumptionActor; +import org.sunbird.viewer.actor.ViewerAggregatorActor; +import org.sunbird.viewer.actor.ViewerSummaryActor; + +public enum ACTOR_NAMES { + VIEW_CONSUMPTION_ACTOR(ViewConsumptionActor.class, "view-consumption-actor"), + VIEWER_AGGREGATOR_ACTOR(ViewerAggregatorActor.class, "viewer-aggregator-actor"), + VIEWER_SUMMARY_ACTOR(ViewerSummaryActor.class, "viewer-summary-actor"); + + private ACTOR_NAMES(Class clazz, String name) { + actorClass = clazz; + actorName = name; + } + + private Class actorClass; + private String actorName; + + public Class getActorClass() { + return actorClass; + } + + public String getActorName() { + return actorName; + } +} diff --git a/modules/viewer/service/app/util/Attrs.java b/modules/viewer/service/app/util/Attrs.java new file mode 100644 index 00000000..dac279a3 --- /dev/null +++ b/modules/viewer/service/app/util/Attrs.java @@ -0,0 +1,22 @@ +package util; + +import org.sunbird.keys.JsonKey; +import play.libs.typedmap.TypedKey; + +public class Attrs { + public static final TypedKey USER_ID = TypedKey.create(JsonKey.USER_ID); + public static final TypedKey AUTH_WITH_MASTER_KEY = TypedKey.create(JsonKey.AUTH_WITH_MASTER_KEY); + public static final TypedKey REQUEST_ID = TypedKey.create(JsonKey.REQUEST_ID); + public static final TypedKey CONTEXT = TypedKey.create(JsonKey.CONTEXT); + public static final TypedKey REQUESTED_FOR = TypedKey.create(JsonKey.REQUESTED_FOR); + public static final TypedKey IS_AUTH_REQ = TypedKey.create(JsonKey.IS_AUTH_REQ); + public static final TypedKey SIGNUP_TYPE = TypedKey.create(JsonKey.SIGNUP_TYPE); + public static final TypedKey REQUEST_SOURCE = TypedKey.create(JsonKey.REQUEST_SOURCE); + public static final TypedKey CHANNEL = TypedKey.create(JsonKey.CHANNEL); + public static final TypedKey APP_ID = TypedKey.create(JsonKey.APP_ID); + public static final TypedKey DEVICE_ID = TypedKey.create(JsonKey.DEVICE_ID); + public static final TypedKey ACTOR_ID = TypedKey.create(JsonKey.ACTOR_ID); + public static final TypedKey ACTOR_TYPE = TypedKey.create(JsonKey.ACTOR_TYPE); + public static final TypedKey X_AUTH_TOKEN = TypedKey.create(JsonKey.X_AUTH_TOKEN); + public static final TypedKey X_LOGGING_HEADERS = TypedKey.create(JsonKey.X_LOGGING_HEADERS); +} diff --git a/modules/viewer/service/app/util/AuthenticationHelper.java b/modules/viewer/service/app/util/AuthenticationHelper.java new file mode 100644 index 00000000..5249a992 --- /dev/null +++ b/modules/viewer/service/app/util/AuthenticationHelper.java @@ -0,0 +1,81 @@ +package util; + +import org.sunbird.auth.verifier.Base64Util; +import org.sunbird.cassandra.CassandraOperation; +import org.sunbird.response.Response; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.helper.ServiceFactory; +import org.sunbird.learner.util.Util; +import org.sunbird.learner.util.Util.DbInfo; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This class will handle all the method related to authentication. For example verifying user + * access token, creating access token after success login. + * + * @author Manzarul + */ +public class AuthenticationHelper { + + private static CassandraOperation cassandraOperation = ServiceFactory.getInstance(); + private static DbInfo userAuth = Util.dbInfoMap.get(JsonKey.USER_AUTH_DB); + public static LoggerUtil logger = new LoggerUtil(AuthenticationHelper.class); + + /** + * This method will verify the incoming user access token against store data base /cache. If token + * is valid then it would be associated with some user id. In case of token matched it will + * provide user id. else will provide empty string. + * + * @param token String + * @return String + */ + @SuppressWarnings("unchecked") + public static String verifyUserAccessToken(String token) { + String userId = JsonKey.UNAUTHORIZED; + try { + Response authResponse = + cassandraOperation.getRecordByIdentifier(userAuth.getKeySpace(), userAuth.getTableName(), token, null, null); + if (authResponse != null && authResponse.get(JsonKey.RESPONSE) != null) { + List> authList = + (List>) authResponse.get(JsonKey.RESPONSE); + if (authList != null && !authList.isEmpty()) { + Map authMap = authList.get(0); + userId = (String) authMap.get(JsonKey.USER_ID); + } + } + } catch (Exception e) { + logger.error("invalid auth token =" + token, e); + } + return userId; + } + + @SuppressWarnings("unchecked") + public static String verifyClientAccessToken(String clientId, String clientToken) { + Util.DbInfo clientDbInfo = Util.dbInfoMap.get(JsonKey.CLIENT_INFO_DB); + Map propertyMap = new HashMap<>(); + propertyMap.put(JsonKey.ID, clientId); + propertyMap.put(JsonKey.MASTER_KEY, clientToken); + String validClientId = JsonKey.UNAUTHORIZED; + try { + Response clientResponse = + cassandraOperation.getRecordsByProperties( + clientDbInfo.getKeySpace(), clientDbInfo.getTableName(), propertyMap, null); + if (null != clientResponse && !clientResponse.getResult().isEmpty()) { + List> dataList = + (List>) clientResponse.getResult().get(JsonKey.RESPONSE); + validClientId = (String) dataList.get(0).get(JsonKey.ID); + } + } catch (Exception e) { + logger.error("Validating client token failed due to : ", e); + } + return validClientId; + } + + private static byte[] decodeFromBase64(String data) { + return Base64Util.decode(data, 11); + } +} diff --git a/modules/viewer/service/app/util/Common.java b/modules/viewer/service/app/util/Common.java new file mode 100644 index 00000000..753902d7 --- /dev/null +++ b/modules/viewer/service/app/util/Common.java @@ -0,0 +1,16 @@ +package util; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Common { + + public static Map getRequestHeadersInArray(Map> requestHeaders) { + Map requestHeadersArray = new HashMap(); + requestHeaders.entrySet().forEach(entry -> { + requestHeadersArray.put(entry.getKey(), (String[]) entry.getValue().toArray(new String[0])); + }); + return requestHeadersArray; + } +} diff --git a/modules/viewer/service/app/util/RequestInterceptor.java b/modules/viewer/service/app/util/RequestInterceptor.java new file mode 100644 index 00000000..ac69db4a --- /dev/null +++ b/modules/viewer/service/app/util/RequestInterceptor.java @@ -0,0 +1,145 @@ +package util; + +import org.apache.commons.lang3.StringUtils; +import org.sunbird.auth.verifier.AccessTokenValidator; +import org.sunbird.keys.JsonKey; +import org.sunbird.logging.LoggerUtil; +import org.sunbird.request.HeaderParam; +import play.mvc.Http; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Request interceptor responsible to authenticated HTTP requests + * + * @author Amit Kumar + */ +public class RequestInterceptor { + + public static List restrictedUriList = null; + private static ConcurrentHashMap apiHeaderIgnoreMap = new ConcurrentHashMap<>(); + private static LoggerUtil logger = new LoggerUtil(RequestInterceptor.class); + + private RequestInterceptor() {} + + static { + restrictedUriList = new ArrayList<>(); + restrictedUriList.add("/v1/content/state/update"); + + short var = 1; + apiHeaderIgnoreMap.put("/service/health", var); + apiHeaderIgnoreMap.put("/v1/page/assemble", var); + apiHeaderIgnoreMap.put("/v1/dial/assemble", var); + apiHeaderIgnoreMap.put("/health", var); + apiHeaderIgnoreMap.put("/v1/data/sync", var); + apiHeaderIgnoreMap.put("/v1/content/link", var); + apiHeaderIgnoreMap.put("/v1/content/unlink", var); + apiHeaderIgnoreMap.put("/v1/content/link/search", var); + apiHeaderIgnoreMap.put("/v1/course/batch/search", var); + apiHeaderIgnoreMap.put("/v1/cache/clear", var); + apiHeaderIgnoreMap.put("/private/v1/course/batch/create", var); + apiHeaderIgnoreMap.put("/v1/course/create", var); + apiHeaderIgnoreMap.put("/v2/user/courses/list", var); + apiHeaderIgnoreMap.put("/v1/collection/summary", var); + } + + /** + * Authenticates given HTTP request context + * + * @param request HTTP play request + * @return User or Client ID for authenticated request. For unauthenticated requests, UNAUTHORIZED + * is returned + */ + public static String verifyRequestData(Http.Request request) { + String clientId = JsonKey.UNAUTHORIZED; + Optional accessToken = request.header(HeaderParam.X_Authenticated_User_Token.getName()); + Optional authClientToken = + request.header(HeaderParam.X_Authenticated_Client_Token.getName()); + Optional authClientId = request.header(HeaderParam.X_Authenticated_Client_Id.getName()); + if (!isRequestInExcludeList(request.path()) && !isRequestPrivate(request.path())) { + if (accessToken.isPresent()) { + // This is to handle Mobile App expired token for content state update API. + if (StringUtils.contains(request.path(), "v1/content/state/update")) { + clientId = AccessTokenValidator.verifyUserToken(accessToken.get(), false); + } else { + clientId = AccessTokenValidator.verifyUserToken(accessToken.get(), true); + } + } else if (authClientToken.isPresent() && authClientId.isPresent()) { + clientId = + AuthenticationHelper.verifyClientAccessToken(authClientId.get(), authClientToken.get()); + if (!JsonKey.UNAUTHORIZED.equals(clientId)) { + request = request.addAttr(Attrs.AUTH_WITH_MASTER_KEY, Boolean.toString(true)); + } + } + return clientId; + } else { + if (accessToken.isPresent()) { + String clientAccessTokenId = null; + try { + // This is to handle Mobile App expired token for content state update API. + if (StringUtils.contains(request.path(), "v1/content/state/update")) { + clientAccessTokenId = AccessTokenValidator.verifyUserToken(accessToken.get(), false); + } else { + clientAccessTokenId = AccessTokenValidator.verifyUserToken(accessToken.get(), true); + } + if (JsonKey.UNAUTHORIZED.equalsIgnoreCase(clientAccessTokenId)) { + clientAccessTokenId = null; + } + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + clientAccessTokenId = null; + } + return StringUtils.isNotBlank(clientAccessTokenId) + ? clientAccessTokenId + : JsonKey.ANONYMOUS; + } + return JsonKey.ANONYMOUS; + } + } + + private static boolean isRequestPrivate(String path) { + return path.contains(JsonKey.PRIVATE); + } + + /** + * Checks if request URL is in excluded (i.e. public) URL list or not + * + * @param requestUrl Request URL + * @return True if URL is in excluded (public) URLs. Otherwise, returns false + */ + public static boolean isRequestInExcludeList(String requestUrl) { + boolean resp = false; + if (!StringUtils.isBlank(requestUrl)) { + if (apiHeaderIgnoreMap.containsKey(requestUrl)) { + resp = true; + } else { + String[] splitPath = requestUrl.split("[/]"); + String urlWithoutPathParam = removeLastValue(splitPath); + if (apiHeaderIgnoreMap.containsKey(urlWithoutPathParam)) { + resp = true; + } + } + } + return resp; + } + + /** + * Returns URL without path and query parameters. + * + * @param splitPath URL path split on slash (i.e. /) + * @return URL without path and query parameters + */ + private static String removeLastValue(String splitPath[]) { + + StringBuilder builder = new StringBuilder(); + if (splitPath != null && splitPath.length > 0) { + for (int i = 1; i < splitPath.length - 1; i++) { + builder.append("/" + splitPath[i]); + } + } + return builder.toString(); + } +} diff --git a/modules/viewer/service/app/util/RequestValidator.java b/modules/viewer/service/app/util/RequestValidator.java new file mode 100644 index 00000000..5f359085 --- /dev/null +++ b/modules/viewer/service/app/util/RequestValidator.java @@ -0,0 +1,1020 @@ +package util; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.telemetry.dto.*; +import org.sunbird.common.ProjectUtil.ProgressStatus; +import org.sunbird.common.ProjectUtil.Source; +import org.sunbird.keys.JsonKey; +import org.sunbird.common.ProjectUtil; +import org.sunbird.common.PropertiesCache; +import org.sunbird.utils.StringFormatter; +import org.sunbird.request.Request; +import org.sunbird.response.ResponseCode; +import org.sunbird.response.ResponseMessage; +import org.sunbird.logging.LoggerUtil; + +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * This call will do validation for all incoming request data. + * + * @author Manzarul + */ +public final class RequestValidator { + private static final int ERROR_CODE = ResponseCode.CLIENT_ERROR.getResponseCode(); + public static LoggerUtil logger = new LoggerUtil(RequestValidator.class); + + private RequestValidator() {} + + /** + * This method will do content state request data validation. if all mandatory data is coming then + * it won't do any thing if any mandatory data is missing then it will throw exception. + * + * @param contentRequestDto Request + */ + @SuppressWarnings("unchecked") + public static void validateUpdateContent(Request contentRequestDto) { + List> list = + (List>) (contentRequestDto.getRequest().get(JsonKey.CONTENTS)); + if(CollectionUtils.isNotEmpty(list)) { + for (Map map : list) { + if (null != map.get(JsonKey.LAST_UPDATED_TIME)) { + boolean bool = + ProjectUtil.isDateValidFormat( + "yyyy-MM-dd HH:mm:ss:SSSZ", (String) map.get(JsonKey.LAST_UPDATED_TIME)); + if (!bool) { + throw new ProjectCommonException( + ResponseCode.dateFormatError, + ResponseCode.dateFormatError.getErrorMessage(), + ERROR_CODE); + } + } + if (null != map.get(JsonKey.LAST_COMPLETED_TIME)) { + boolean bool = + ProjectUtil.isDateValidFormat( + "yyyy-MM-dd HH:mm:ss:SSSZ", (String) map.get(JsonKey.LAST_COMPLETED_TIME)); + if (!bool) { + throw new ProjectCommonException( + ResponseCode.dateFormatError, + ResponseCode.dateFormatError.getErrorMessage(), + ERROR_CODE); + } + } + String courseId = map.containsKey(JsonKey.COURSE_ID) ? JsonKey.COURSE_ID : JsonKey.COLLECTION_ID; + map.put(JsonKey.COURSE_ID, map.get(courseId)); + if (StringUtils.isBlank((String) map.get(JsonKey.COURSE_ID))) { + throw new ProjectCommonException( + ResponseCode.courseIdRequired, + ResponseCode.courseIdRequiredError.getErrorMessage(), + ERROR_CODE); + } + if (map.containsKey(JsonKey.CONTENT_ID)) { + + if (null == map.get(JsonKey.CONTENT_ID)) { + throw new ProjectCommonException( + ResponseCode.contentIdRequired, + ResponseCode.contentIdRequiredError.getErrorMessage(), + ERROR_CODE); + } + if (ProjectUtil.isNull(map.get(JsonKey.STATUS))) { + throw new ProjectCommonException( + ResponseCode.contentStatusRequired, + ResponseCode.contentStatusRequired.getErrorMessage(), + ERROR_CODE); + } + + } else { + throw new ProjectCommonException( + ResponseCode.contentIdRequired, + ResponseCode.contentIdRequiredError.getErrorMessage(), + ERROR_CODE); + } + } + } + List> assessmentData = + (List>) contentRequestDto.getRequest().get(JsonKey.ASSESSMENT_EVENTS); + if (CollectionUtils.isNotEmpty(assessmentData)) { + for (Map map : assessmentData) { + if (!map.containsKey(JsonKey.ASSESSMENT_TS)) { + throw new ProjectCommonException( + ResponseCode.assessmentAttemptDateRequired, + ResponseCode.assessmentAttemptDateRequired.getErrorMessage(), + ERROR_CODE); + } + + if (!map.containsKey(JsonKey.COURSE_ID) + || StringUtils.isBlank((String) map.get(JsonKey.COURSE_ID))) { + throw new ProjectCommonException( + ResponseCode.courseIdRequired, + ResponseCode.courseIdRequiredError.getErrorMessage(), + ERROR_CODE); + } + + if (!map.containsKey(JsonKey.CONTENT_ID) + || StringUtils.isBlank((String) map.get(JsonKey.CONTENT_ID))) { + throw new ProjectCommonException( + ResponseCode.contentIdRequired, + ResponseCode.contentIdRequiredError.getErrorMessage(), + ERROR_CODE); + } + + if (!map.containsKey(JsonKey.BATCH_ID) + || StringUtils.isBlank((String) map.get(JsonKey.BATCH_ID))) { + throw new ProjectCommonException( + ResponseCode.courseBatchIdRequired, + ResponseCode.courseBatchIdRequired.getErrorMessage(), + ERROR_CODE); + } + + if (!map.containsKey(JsonKey.USER_ID) + || StringUtils.isBlank((String) map.get(JsonKey.USER_ID))) { + throw new ProjectCommonException( + ResponseCode.userIdRequired, + ResponseCode.userIdRequired.getErrorMessage(), + ERROR_CODE); + } + + if (!map.containsKey(JsonKey.ATTEMPT_ID) + || StringUtils.isBlank((String) map.get(JsonKey.ATTEMPT_ID))) { + throw new ProjectCommonException( + ResponseCode.attemptIdRequired, + ResponseCode.attemptIdRequired.getErrorMessage(), + ERROR_CODE); + } + + if (!map.containsKey(JsonKey.EVENTS)) { + throw new ProjectCommonException( + ResponseCode.eventsRequired, + ResponseCode.eventsRequired.getErrorMessage(), + ERROR_CODE); + } + } + } + // Validation for enrolment sync + if(CollectionUtils.isEmpty(list) && CollectionUtils.isEmpty(assessmentData)) { + contentRequestDto.getRequest().put(JsonKey.COURSE_ID, contentRequestDto.getOrDefault(JsonKey.COURSE_ID, contentRequestDto.getOrDefault(JsonKey.COLLECTION_ID, ""))); + if (StringUtils.isBlank((String) contentRequestDto.getOrDefault(JsonKey.COURSE_ID, ""))) { + throw new ProjectCommonException( + ResponseCode.courseIdRequired, + ResponseCode.courseIdRequiredError.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank((String) contentRequestDto.getOrDefault(JsonKey.BATCH_ID, ""))) { + throw new ProjectCommonException( + ResponseCode.courseBatchIdRequired, + ResponseCode.courseBatchIdRequired.getErrorMessage(), + ERROR_CODE); + } + + if (StringUtils.isBlank((String) contentRequestDto.getOrDefault(JsonKey.USER_ID, ""))) { + throw new ProjectCommonException( + ResponseCode.userIdRequired, + ResponseCode.userIdRequired.getErrorMessage(), + ERROR_CODE); + } + } + } + + /** + * This method will validate get page data api. + * + * @param request Request + */ + public static void validateGetPageData(Request request) { + if (request == null || (StringUtils.isBlank((String) request.get(JsonKey.SOURCE)))) { + throw new ProjectCommonException( + ResponseCode.sourceRequired, + ResponseCode.sourceRequired.getErrorMessage(), + ERROR_CODE); + } + if (!validPageSourceType((String) request.get(JsonKey.SOURCE))) { + throw new ProjectCommonException( + ResponseCode.invalidPageSource, + ResponseCode.invalidPageSource.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank((String) request.get(JsonKey.PAGE_NAME))) { + throw new ProjectCommonException( + ResponseCode.pageNameRequired, + ResponseCode.pageNameRequired.getErrorMessage(), + ERROR_CODE); + } + } + + private static boolean validPageSourceType(String source) { + + Boolean isValidSource = false; + for (Source src : Source.values()) { + if (src.getValue().equalsIgnoreCase(source)) { + isValidSource = true; + break; + } + } + return isValidSource; + } + + /** + * This method will validate add course request data. + * + * @param courseRequest Request + */ + public static void validateAddBatchCourse(Request courseRequest) { + + if (courseRequest.getRequest().get(JsonKey.BATCH_ID) == null) { + throw new ProjectCommonException( + ResponseCode.courseBatchIdRequired, + ResponseCode.courseBatchIdRequired.getErrorMessage(), + ERROR_CODE); + } + if (courseRequest.getRequest().get(JsonKey.USER_IDs) == null) { + throw new ProjectCommonException( + ResponseCode.userIdRequired, + ResponseCode.userIdRequired.getErrorMessage(), + ERROR_CODE); + } + } + + /** + * This method will validate add course request data. + * + * @param courseRequest Request + */ + public static void validateGetBatchCourse(Request courseRequest) { + + if (courseRequest.getRequest().get(JsonKey.BATCH_ID) == null) { + throw new ProjectCommonException( + ResponseCode.courseBatchIdRequired, + ResponseCode.courseBatchIdRequired.getErrorMessage(), + ERROR_CODE); + } + } + + /** + * This method will validate update course request data. + * + * @param request Request + */ + public static void validateUpdateCourse(Request request) { + + if (request.getRequest().get(JsonKey.COURSE_ID) == null) { + throw new ProjectCommonException( + ResponseCode.courseIdRequired, + ResponseCode.courseIdRequired.getErrorMessage(), + ERROR_CODE); + } + } + + /** + * This method will validate published course request data. + * + * @param request Request + */ + public static void validatePublishCourse(Request request) { + if (request.getRequest().get(JsonKey.COURSE_ID) == null) { + throw new ProjectCommonException( + ResponseCode.courseIdRequiredError, + ResponseCode.courseIdRequiredError.getErrorMessage(), + ERROR_CODE); + } + } + + /** + * This method will validate Delete course request data. + * + * @param request Request + */ + public static void validateDeleteCourse(Request request) { + if (request.getRequest().get(JsonKey.COURSE_ID) == null) { + throw new ProjectCommonException( + ResponseCode.courseIdRequiredError, + ResponseCode.courseIdRequiredError.getErrorMessage(), + ERROR_CODE); + } + } + + /* + * This method will validate create section data + * + * @param userRequest Request + */ + public static void validateCreateSection(Request request) { + if (StringUtils.isBlank( + (String) + (request.getRequest().get(JsonKey.SECTION_NAME) != null + ? request.getRequest().get(JsonKey.SECTION_NAME) + : ""))) { + throw new ProjectCommonException( + ResponseCode.sectionNameRequired, + ResponseCode.sectionNameRequired.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank( + (String) + (request.getRequest().get(JsonKey.SECTION_DATA_TYPE) != null + ? request.getRequest().get(JsonKey.SECTION_DATA_TYPE) + : ""))) { + throw new ProjectCommonException( + ResponseCode.sectionDataTypeRequired, + ResponseCode.sectionDataTypeRequired.getErrorMessage(), + ERROR_CODE); + } + } + + /** + * This method will validate update section request data + * + * @param request Request + */ + public static void validateUpdateSection(Request request) { + if (request.getRequest().containsKey(JsonKey.SECTION_NAME) + && StringUtils.isBlank( + (String) + (request.getRequest().get(JsonKey.SECTION_NAME) != null + ? request.getRequest().get(JsonKey.SECTION_NAME) + : ""))) { + throw new ProjectCommonException( + ResponseCode.sectionNameRequired, + ResponseCode.sectionNameRequired.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank( + (String) + (request.getRequest().get(JsonKey.ID) != null + ? request.getRequest().get(JsonKey.ID) + : ""))) { + throw new ProjectCommonException( + ResponseCode.sectionIdRequired, + ResponseCode.sectionIdRequired.getErrorMessage(), + ERROR_CODE); + } + if (request.getRequest().containsKey(JsonKey.SECTION_DATA_TYPE) + && StringUtils.isBlank( + (String) + (request.getRequest().get(JsonKey.SECTION_DATA_TYPE) != null + ? request.getRequest().get(JsonKey.SECTION_DATA_TYPE) + : ""))) { + throw new ProjectCommonException( + ResponseCode.sectionDataTypeRequired, + ResponseCode.sectionDataTypeRequired.getErrorMessage(), + ERROR_CODE); + } + } + + /** + * This method will validate create page data + * + * @param request Request + */ + public static void validateCreatePage(Request request) { + if (StringUtils.isEmpty( + (String) + (request.getRequest().get(JsonKey.PAGE_NAME) != null + ? request.getRequest().get(JsonKey.PAGE_NAME) + : ""))) { + throw new ProjectCommonException( + ResponseCode.pageNameRequired, + ResponseCode.pageNameRequired.getErrorMessage(), + ERROR_CODE); + } + } + + /** + * This method will validate update page request data + * + * @param request Request + */ + public static void validateUpdatepage(Request request) { + if (request.getRequest().containsKey(JsonKey.PAGE_NAME) + && StringUtils.isEmpty( + (String) + (request.getRequest().get(JsonKey.PAGE_NAME) != null + ? request.getRequest().get(JsonKey.PAGE_NAME) + : ""))) { + throw new ProjectCommonException( + ResponseCode.pageNameRequired, + ResponseCode.pageNameRequired.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank( + (String) + (request.getRequest().get(JsonKey.ID) != null + ? request.getRequest().get(JsonKey.ID) + : ""))) { + throw new ProjectCommonException( + ResponseCode.pageIdRequired, + ResponseCode.pageIdRequired.getErrorMessage(), + ERROR_CODE); + } + } + + /** + * This method will validate bulk user upload requested data. + * + * @param reqObj Request + */ + public static void validateUploadUser(Map reqObj) { + if (StringUtils.isBlank((String) reqObj.get(JsonKey.ORGANISATION_ID)) + && (StringUtils.isBlank((String) reqObj.get(JsonKey.ORG_EXTERNAL_ID)) + || StringUtils.isBlank((String) reqObj.get(JsonKey.ORG_PROVIDER)))) { + throw new ProjectCommonException( + ResponseCode.mandatoryParamsMissing, + ProjectUtil.formatMessage( + ResponseCode.mandatoryParamsMissing.getErrorMessage(), + (ProjectUtil.formatMessage( + ResponseMessage.Message.OR_FORMAT, + JsonKey.ORGANISATION_ID, + ProjectUtil.formatMessage( + ResponseMessage.Message.AND_FORMAT, + JsonKey.ORG_EXTERNAL_ID, + JsonKey.ORG_PROVIDER)))), + ERROR_CODE); + } + if (null == reqObj.get(JsonKey.FILE)) { + throw new ProjectCommonException( + ResponseCode.mandatoryParamsMissing, + ProjectUtil.formatMessage( + ResponseCode.mandatoryParamsMissing.getErrorMessage(), JsonKey.FILE), + ERROR_CODE); + } + } + + /** + * courseId : Should be a valid courseId under EKStep. name : should not be null or empty + * enrolmentType: can have only following two values {"open","invite-only"} startDate : In + * yyyy-MM-DD format , and must be >= today date. endDate : In yyyy-MM-DD format and must be > + * startDate createdFor : List of valid organisation ids. this filed will be used in case of + * "invite-only" enrolmentType. for open type if createdFor values is coming then system will just + * save that value. mentors : List of user ids , who will work as a mentor. + * + * @param request + */ + public static void validateCreateBatchReq(Request request) { + if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.COURSE_ID))) { + throw new ProjectCommonException( + ResponseCode.invalidCourseId, + ResponseCode.invalidCourseId.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.NAME))) { + throw new ProjectCommonException( + ResponseCode.courseNameRequired, + ResponseCode.courseNameRequired.getErrorMessage(), + ERROR_CODE); + } + String enrolmentType = (String) request.getRequest().get(JsonKey.ENROLLMENT_TYPE); + validateEnrolmentType(enrolmentType); + String startDate = (String) request.getRequest().get(JsonKey.START_DATE); + String endDate = (String) request.getRequest().get(JsonKey.END_DATE); + validateStartDate(startDate); + validateEndDate(startDate, endDate); + + if (request.getRequest().containsKey(JsonKey.COURSE_CREATED_FOR) + && !(request.getRequest().get(JsonKey.COURSE_CREATED_FOR) instanceof List)) { + throw new ProjectCommonException( + ResponseCode.dataTypeError, + ResponseCode.dataTypeError.getErrorMessage(), + ERROR_CODE); + } + } + + private static boolean checkProgressStatus(int status) { + for (ProgressStatus pstatus : ProgressStatus.values()) { + if (pstatus.getValue() == status) { + return true; + } + } + return false; + } + + public static void validateUpdateCourseBatchReq(Request request) { + + if (null != request.getRequest().get(JsonKey.STATUS)) { + boolean status = validateBatchStatus(request); + if (!status) { + throw new ProjectCommonException( + ResponseCode.progressStatusError, + ResponseCode.progressStatusError.getErrorMessage(), + ERROR_CODE); + } + } + if (request.getRequest().containsKey(JsonKey.NAME) + && StringUtils.isEmpty((String) request.getRequest().get(JsonKey.NAME))) { + throw new ProjectCommonException( + ResponseCode.courseNameRequired, + ResponseCode.courseNameRequired.getErrorMessage(), + ERROR_CODE); + } + if (request.getRequest().containsKey(JsonKey.ENROLLMENT_TYPE)) { + String enrolmentType = (String) request.getRequest().get(JsonKey.ENROLLMENT_TYPE); + validateEnrolmentType(enrolmentType); + } + String startDate = (String) request.getRequest().get(JsonKey.START_DATE); + String endDate = (String) request.getRequest().get(JsonKey.END_DATE); + + validateUpdateBatchStartDate(startDate); + validateEndDate(startDate, endDate); + + boolean bool = validateDateWithTodayDate(endDate); + if (!bool) { + throw new ProjectCommonException( + ResponseCode.invalidBatchEndDateError, + ResponseCode.invalidBatchEndDateError.getErrorMessage(), + ERROR_CODE); + } + + validateUpdateBatchEndDate(request); + if (request.getRequest().containsKey(JsonKey.COURSE_CREATED_FOR) + && !(request.getRequest().get(JsonKey.COURSE_CREATED_FOR) instanceof List)) { + throw new ProjectCommonException( + ResponseCode.dataTypeError, + ResponseCode.dataTypeError.getErrorMessage(), + ERROR_CODE); + } + + if (request.getRequest().containsKey(JsonKey.MENTORS) + && !(request.getRequest().get(JsonKey.MENTORS) instanceof List)) { + throw new ProjectCommonException( + ResponseCode.dataTypeError, + ResponseCode.dataTypeError.getErrorMessage(), + ERROR_CODE); + } + } + + private static void validateUpdateBatchStartDate(String startDate) { + if (StringUtils.isNotBlank(startDate)) { + try { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + format.parse(startDate); + } catch (Exception e) { + throw new ProjectCommonException( + ResponseCode.dateFormatError, + ResponseCode.dateFormatError.getErrorMessage(), + ERROR_CODE); + } + } else { + throw new ProjectCommonException( + ResponseCode.courseBatchStartDateRequired, + ResponseCode.courseBatchStartDateRequired.getErrorMessage(), + ERROR_CODE); + } + } + + private static boolean validateBatchStatus(Request request) { + boolean status = false; + try { + status = checkProgressStatus(Integer.parseInt("" + request.getRequest().get(JsonKey.STATUS))); + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return status; + } + + private static void validateUpdateBatchEndDate(Request request) { + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + String startDate = (String) request.getRequest().get(JsonKey.START_DATE); + String endDate = (String) request.getRequest().get(JsonKey.END_DATE); + format.setLenient(false); + if (StringUtils.isNotBlank(endDate) && StringUtils.isNotBlank(startDate)) { + Date batchStartDate = null; + Date batchEndDate = null; + try { + batchStartDate = format.parse(startDate); + batchEndDate = format.parse(endDate); + Calendar cal1 = Calendar.getInstance(); + Calendar cal2 = Calendar.getInstance(); + cal1.setTime(batchStartDate); + cal2.setTime(batchEndDate); + } catch (Exception e) { + throw new ProjectCommonException( + ResponseCode.dateFormatError, + ResponseCode.dateFormatError.getErrorMessage(), + ERROR_CODE); + } + if (batchEndDate.before(batchStartDate)) { + throw new ProjectCommonException( + ResponseCode.invalidBatchEndDateError, + ResponseCode.invalidBatchEndDateError.getErrorMessage(), + ERROR_CODE); + } + } + } + + private static boolean validateDateWithTodayDate(String date) { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + format.setLenient(false); + try { + if (StringUtils.isNotEmpty(date)) { + Date reqDate = format.parse(date); + Date todayDate = format.parse(format.format(new Date())); + Calendar cal1 = Calendar.getInstance(); + Calendar cal2 = Calendar.getInstance(); + cal1.setTime(reqDate); + cal2.setTime(todayDate); + if (reqDate.before(todayDate)) { + return false; + } + } + } catch (Exception e) { + throw new ProjectCommonException( + ResponseCode.dateFormatError, + ResponseCode.dateFormatError.getErrorMessage(), + ERROR_CODE); + } + return true; + } + + /** @param enrolmentType */ + public static void validateEnrolmentType(String enrolmentType) { + if (StringUtils.isBlank(enrolmentType)) { + throw new ProjectCommonException( + ResponseCode.enrolmentTypeRequired, + ResponseCode.enrolmentTypeRequired.getErrorMessage(), + ERROR_CODE); + } + if (!(ProjectUtil.EnrolmentType.open.getVal().equalsIgnoreCase(enrolmentType) + || ProjectUtil.EnrolmentType.inviteOnly.getVal().equalsIgnoreCase(enrolmentType))) { + throw new ProjectCommonException( + ResponseCode.enrolmentIncorrectValue, + ResponseCode.enrolmentIncorrectValue.getErrorMessage(), + ERROR_CODE); + } + } + + /** @param startDate */ + private static void validateStartDate(String startDate) { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + format.setLenient(false); + if (StringUtils.isBlank(startDate)) { + throw new ProjectCommonException( + ResponseCode.courseBatchStartDateRequired, + ResponseCode.courseBatchStartDateRequired.getErrorMessage(), + ERROR_CODE); + } + try { + Date batchStartDate = format.parse(startDate); + Date todayDate = format.parse(format.format(new Date())); + Calendar cal1 = Calendar.getInstance(); + Calendar cal2 = Calendar.getInstance(); + cal1.setTime(batchStartDate); + cal2.setTime(todayDate); + if (batchStartDate.before(todayDate)) { + throw new ProjectCommonException( + ResponseCode.courseBatchStartDateError, + ResponseCode.courseBatchStartDateError.getErrorMessage(), + ERROR_CODE); + } + } catch (ProjectCommonException e) { + throw e; + } catch (Exception e) { + throw new ProjectCommonException( + ResponseCode.dateFormatError, + ResponseCode.dateFormatError.getErrorMessage(), + ERROR_CODE); + } + } + + private static void validateEndDate(String startDate, String endDate) { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + format.setLenient(false); + Date batchEndDate = null; + Date batchStartDate = null; + try { + if (StringUtils.isNotEmpty(endDate)) { + batchEndDate = format.parse(endDate); + batchStartDate = format.parse(startDate); + } + } catch (Exception e) { + throw new ProjectCommonException( + ResponseCode.dateFormatError, + ResponseCode.dateFormatError.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isNotEmpty(endDate) && batchStartDate.getTime() >= batchEndDate.getTime()) { + throw new ProjectCommonException( + ResponseCode.endDateError, + ResponseCode.endDateError.getErrorMessage(), + ERROR_CODE); + } + } + + public static void validateSyncRequest(Request request) { + String operation = (String) request.getRequest().get(JsonKey.OPERATION_FOR); + if ((null != operation) && (!operation.equalsIgnoreCase("keycloak"))) { + if (request.getRequest().get(JsonKey.OBJECT_TYPE) == null) { + throw new ProjectCommonException( + ResponseCode.dataTypeError, + ResponseCode.dataTypeError.getErrorMessage(), + ERROR_CODE); + } + List list = + new ArrayList<>( + Arrays.asList( + new String[] { + JsonKey.USER, JsonKey.ORGANISATION, JsonKey.BATCH, JsonKey.USER_COURSE + })); + if (!list.contains(request.getRequest().get(JsonKey.OBJECT_TYPE))) { + throw new ProjectCommonException( + ResponseCode.invalidObjectType, + ResponseCode.invalidObjectType.getErrorMessage(), + ERROR_CODE); + } + } + } + + public static void validateUpdateSystemSettingsRequest(Request request) { + List list = + new ArrayList<>( + Arrays.asList( + PropertiesCache.getInstance() + .getProperty("system_settings_properties") + .split(","))); + for (String str : request.getRequest().keySet()) { + if (!list.contains(str)) { + throw new ProjectCommonException( + ResponseCode.invalidPropertyError, + MessageFormat.format(ResponseCode.invalidPropertyError.getErrorMessage(), str), + ERROR_CODE); + } + } + } + + public static void validateSendMail(Request request) { + if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.SUBJECT))) { + throw new ProjectCommonException( + ResponseCode.emailSubjectError, + ResponseCode.emailSubjectError.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.BODY))) { + throw new ProjectCommonException( + ResponseCode.emailBodyError, + ResponseCode.emailBodyError.getErrorMessage(), + ERROR_CODE); + } + if (CollectionUtils.isEmpty((List) (request.getRequest().get(JsonKey.RECIPIENT_EMAILS))) + && CollectionUtils.isEmpty( + (List) (request.getRequest().get(JsonKey.RECIPIENT_USERIDS))) + && MapUtils.isEmpty( + (Map) (request.getRequest().get(JsonKey.RECIPIENT_SEARCH_QUERY))) + && CollectionUtils.isEmpty( + (List) (request.getRequest().get(JsonKey.RECIPIENT_PHONES)))) { + throw new ProjectCommonException( + ResponseCode.mandatoryParamsMissing, + MessageFormat.format( + ResponseCode.mandatoryParamsMissing.getErrorMessage(), + StringFormatter.joinByOr( + StringFormatter.joinByComma( + JsonKey.RECIPIENT_EMAILS, + JsonKey.RECIPIENT_USERIDS, + JsonKey.RECIPIENT_PHONES), + JsonKey.RECIPIENT_SEARCH_QUERY)), + ERROR_CODE); + } + } + + public static void validateFileUpload(Request reqObj) { + + if (StringUtils.isBlank((String) reqObj.get(JsonKey.CONTAINER))) { + throw new ProjectCommonException( + ResponseCode.storageContainerNameMandatory, + ResponseCode.storageContainerNameMandatory.getErrorMessage(), + ERROR_CODE); + } + } + + /** @param reqObj */ + public static void validateCreateOrgType(Request reqObj) { + if (StringUtils.isBlank((String) reqObj.getRequest().get(JsonKey.NAME))) { + throw createExceptionInstance(ResponseCode.orgTypeMandatory); + } + } + + /** @param reqObj */ + public static void validateUpdateOrgType(Request reqObj) { + if (StringUtils.isBlank((String) reqObj.getRequest().get(JsonKey.NAME))) { + throw createExceptionInstance(ResponseCode.orgTypeMandatory); + } + if (StringUtils.isBlank((String) reqObj.getRequest().get(JsonKey.ID))) { + throw createExceptionInstance(ResponseCode.orgTypeIdRequired); + } + } + + /** + * Method to validate not for userId, title, note, courseId, contentId and tags + * + * @param request + */ + @SuppressWarnings("rawtypes") + public static void validateNote(Request request) { + if (StringUtils.isBlank((String) request.get(JsonKey.USER_ID))) { + throw new ProjectCommonException( + ResponseCode.userIdRequired, + ResponseCode.userIdRequired.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank((String) request.get(JsonKey.TITLE))) { + throw new ProjectCommonException( + ResponseCode.titleRequired, + ResponseCode.titleRequired.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank((String) request.get(JsonKey.NOTE))) { + throw new ProjectCommonException( + ResponseCode.noteRequired, + ResponseCode.noteRequired.getErrorMessage(), + ERROR_CODE); + } + if (StringUtils.isBlank((String) request.get(JsonKey.CONTENT_ID)) + && StringUtils.isBlank((String) request.get(JsonKey.COURSE_ID))) { + throw new ProjectCommonException( + ResponseCode.contentIdError, + ResponseCode.contentIdError.getErrorMessage(), + ERROR_CODE); + } + if (request.getRequest().containsKey(JsonKey.TAGS) + && ((request.getRequest().get(JsonKey.TAGS) instanceof List) + && ((List) request.getRequest().get(JsonKey.TAGS)).isEmpty())) { + throw new ProjectCommonException( + ResponseCode.invalidTags, + ResponseCode.invalidTags.getErrorMessage(), + ERROR_CODE); + } else if (request.getRequest().get(JsonKey.TAGS) instanceof String) { + throw new ProjectCommonException( + ResponseCode.invalidTags, + ResponseCode.invalidTags.getErrorMessage(), + ERROR_CODE); + } + } + + /** + * Method to validate noteId + * + * @param noteId + */ + public static void validateNoteId(String noteId) { + if (StringUtils.isBlank(noteId)) { + throw createExceptionInstance(ResponseCode.invalidNoteId); + } + } + + /** + * Method to validate + * + * @param request + */ + public static void validateRegisterClient(Request request) { + + if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.CLIENT_NAME))) { + throw createExceptionInstance(ResponseCode.invalidClientName); + } + } + + /** + * Method to validate the request for updating the client key + * + * @param clientId + * @param masterAccessToken + */ + public static void validateUpdateClientKey(String clientId, String masterAccessToken) { + validateClientId(clientId); + if (StringUtils.isBlank(masterAccessToken)) { + throw createExceptionInstance(ResponseCode.invalidRequestData); + } + } + + /** + * Method to validate the request for updating the client key + * + * @param id + * @param type + */ + public static void validateGetClientKey(String id, String type) { + validateClientId(id); + if (StringUtils.isBlank(type)) { + throw createExceptionInstance(ResponseCode.invalidRequestData); + } + } + + /** + * Method to validate clientId. + * + * @param clientId + */ + public static void validateClientId(String clientId) { + if (StringUtils.isBlank(clientId)) { + throw createExceptionInstance(ResponseCode.invalidClientId); + } + } + + /** + * Method to validate notification request data. + * + * @param request Request + */ + @SuppressWarnings("unchecked") + public static void validateSendNotification(Request request) { + if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.TO))) { + throw createExceptionInstance(ResponseCode.invalidTopic); + } + if (request.getRequest().get(JsonKey.DATA) == null + || !(request.getRequest().get(JsonKey.DATA) instanceof Map) + || ((Map) request.getRequest().get(JsonKey.DATA)).size() == 0) { + throw createExceptionInstance(ResponseCode.invalidTopicData); + } + + if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.TYPE))) { + throw createExceptionInstance(ResponseCode.invalidNotificationType); + } + if (!(JsonKey.FCM.equalsIgnoreCase((String) request.getRequest().get(JsonKey.TYPE)))) { + throw createExceptionInstance(ResponseCode.notificationTypeSupport); + } + } + + @SuppressWarnings("rawtypes") + public static void validateGetUserCount(Request request) { + if (!validateListType(request, JsonKey.LOCATION_IDS)) { + throw createDataTypeException( + ResponseCode.dataTypeError, JsonKey.LOCATION_IDS, JsonKey.LIST); + } + if (null == request.getRequest().get(JsonKey.LOCATION_IDS) + && ((List) request.getRequest().get(JsonKey.LOCATION_IDS)).isEmpty()) { + throw createExceptionInstance(ResponseCode.locationIdRequired); + } + + if (!validateBooleanType(request, JsonKey.USER_LIST_REQ)) { + throw createDataTypeException( + ResponseCode.dataTypeError, JsonKey.USER_LIST_REQ, "Boolean"); + } + + if (null != request.getRequest().get(JsonKey.USER_LIST_REQ) + && (Boolean) request.getRequest().get(JsonKey.USER_LIST_REQ)) { + throw createExceptionInstance(ResponseCode.functionalityMissing); + } + + if (!validateBooleanType(request, JsonKey.ESTIMATED_COUNT_REQ)) { + throw createDataTypeException( + ResponseCode.dataTypeError, JsonKey.ESTIMATED_COUNT_REQ, "Boolean"); + } + + if (null != request.getRequest().get(JsonKey.ESTIMATED_COUNT_REQ) + && (Boolean) request.getRequest().get(JsonKey.ESTIMATED_COUNT_REQ)) { + throw createExceptionInstance(ResponseCode.functionalityMissing); + } + } + + /** + * if the request contains that key and key is not instance of List then it will return false. + * other cases it will return true. + * + * @param request Request + * @param key String + * @return boolean + */ + private static boolean validateListType(Request request, String key) { + return !(request.getRequest().containsKey(key) + && null != request.getRequest().get(key) + && !(request.getRequest().get(key) instanceof List)); + } + + /** + * If the request contains the key and key value is not Boolean type then it will return false , + * for any other case it will return true. + * + * @param request Request + * @param key String + * @return boolean + */ + private static boolean validateBooleanType(Request request, String key) { + return !(request.getRequest().containsKey(key) + && null != request.getRequest().get(key) + && !(request.getRequest().get(key) instanceof Boolean)); + } + + private static ProjectCommonException createDataTypeException( + ResponseCode responseCode, String key1, String key2) { + return new ProjectCommonException( + responseCode, + ProjectUtil.formatMessage( + responseCode.getErrorMessage(), key1, key2), + ERROR_CODE); + } + + private static ProjectCommonException createExceptionInstance(ResponseCode responseCode) { + return new ProjectCommonException( + responseCode, + responseCode.getErrorMessage(), + ERROR_CODE); + } +} diff --git a/modules/viewer/service/conf/application.conf b/modules/viewer/service/conf/application.conf new file mode 100644 index 00000000..a9e48cec --- /dev/null +++ b/modules/viewer/service/conf/application.conf @@ -0,0 +1,404 @@ +# This is the main configuration file for the application. +# https://www.playframework.com/documentation/latest/ConfigFile +# ~~~~~ + +## Pekko +# https://www.playframework.com/documentation/latest/JavaPekko#Configuration +# ~~~~~ +pekko { + loggers = ["org.apache.pekko.event.slf4j.Slf4jLogger"] + loglevel = "INFO" + stdout-loglevel = "DEBUG" + logging-filter = "org.apache.pekko.event.slf4j.Slf4jLoggingFilter" + log-config-on-start = off + + actor { + provider = "org.apache.pekko.actor.LocalActorRefProvider" + serializers { + java = "org.apache.pekko.serialization.JavaSerializer" + } + serialization-bindings { + "org.sunbird.request.Request" = java + "org.sunbird.response.Response" = java + } + default-dispatcher { + # This will be used if you have set "executor = "fork-join-executor"" + fork-join-executor { + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 8 + + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 32.0 + + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 64 + + # Setting to "FIFO" to use queue like peeking mode which "poll" or "LIFO" to use stack + # like peeking mode which "pop". + task-peeking-mode = "FIFO" + } + } + page-mgr-actor-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + tracking-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + rr-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + brr-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 1 + parallelism-factor = 2.0 + parallelism-max = 4 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + deployment { + /view-consumption-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = tracking-dispatcher + } + # viewer-aggregator-actor is bound programmatically in ActorStartModule with a + # ConsistentHashingPool + hashMapper keyed on userId (per-user serialization). + # No deployment entry here: config FromConfig cannot supply a hash key. + /viewer-summary-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /page-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 10 + dispatcher = page-mgr-actor-dispatcher + } + /cache-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /course-metrics-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /course-enrolment-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = tracking-dispatcher + } + /content-consumption-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = tracking-dispatcher + } + /course-batch-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /search-handler-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /health-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /course-batch-certificate-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /certificate-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /qrcode-download-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /bulk-upload-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /es-sync-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /bulk-upload-background-job-actor + { + router = smallest-mailbox-pool + nr-of-instances = 2 + dispatcher = brr-dispatcher + } + /course-batch-notification-actor + { + router = smallest-mailbox-pool + nr-of-instances = 2 + dispatcher = brr-dispatcher + } + /background-job-manager-actor + { + router = smallest-mailbox-pool + nr-of-instances = 2 + dispatcher = brr-dispatcher + } + /course-management-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /group-aggregates-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /collection-summary-aggregate-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /exhaust-job-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /assessment-aggregator-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /activity-aggregator-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + /observability-report-actor + { + router = smallest-mailbox-pool + nr-of-instances = 4 + dispatcher = rr-dispatcher + } + } + } +} + +## Internationalisation +# https://www.playframework.com/documentation/latest/JavaI18N +# ~~~~~ +play.i18n { + # The application languages + langs = [ "en" ] +} + +## Play HTTP settings +# ~~~~~ +play.http { + ## Router + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # Define the Router object to use for this application. + # This router will be looked up first when the application is starting up, + # so make sure this is the entry point. + # Furthermore, it's assumed your route file is named properly. + # So for an application router like `my.application.Router`, + # you may need to define a router file `conf/my.application.routes`. + # Default to Routes in the root package (aka "apps" folder) (and conf/routes) + #router = my.application.Router + #TO allow more data in request body + errorHandler = modules.ErrorHandler + actionCreator = modules.OnRequestHandler + parser { + maxMemoryBuffer = 50MB + maxDiskBuffer = 50MB + } + secret.key="courseservicenadnfrfnsdmcvdfjbnfrsjfksdvcsdnfvdsfjkrvhsmdv" +} + +## Netty Provider +# ~~~~~ +play.server { + + provider = "play.core.server.NettyServerProvider" + + netty { + # The number of event loop threads. 0 means let Netty decide, which by default will select 2 times the number of + # available processors. + eventLoopThreads = 30 + + log.wire = true + + # The transport to use, either jdk or native. + # Native socket transport has higher performance and produces less garbage but are only available on linux + transport = "native" + + # If you run Play on Linux, you can use Netty's native socket transport + # for higher performance with less garbage. + #transport = "native" + maxChunkSize = 30000000 + option { + + # Set whether connections should use TCP keep alive + child.SO_KEEPALIVE = true + + # Set whether the TCP no delay flag is set + # child.tcpNoDelay = false + + # Set the size of the backlog of TCP connections. The default and exact meaning of this parameter is JDK specific. + # backlog = 100 + } + + http { + # The idle timeout for an open connection after which it will be closed + # Set to null or "infinite" to disable the timeout, but notice that this + # is not encouraged since timeout are important mechanisms to protect your + # servers from malicious attacks or programming mistakes. + idleTimeout = infinite + } + } +} + +## WS (HTTP Client) +# ~~~~~ +libraryDependencies += javaWs + +AuthenticationEnabled=true + +## Cache +# https://www.playframework.com/documentation/latest/JavaCache +# https://www.playframework.com/documentation/latest/ScalaCache +# ~~~~~ +# Play comes with an integrated cache API that can reduce the operational +# overhead of repeated requests. You must enable this by adding to build.sbt: +# +libraryDependencies += cache +# +play.cache { + # If you want to bind several caches, you can bind the individually + #bindCaches = ["db-cache", "user-cache", "session-cache"] +} + +# Logger +# ~~~~~ +# You can also configure logback (http://logback.qos.ch/), +# by providing an application-logger.xml file in the conf directory. + +# Root logger: +#logger.root=ERROR + +# Logger used by the framework: +#logger.play=INFO + +# Logger provided to your application: +#logger.application=DEBUG + +# APP Specific config +# ~~~~~ +# Application specific configurations can be provided here +play.modules { + enabled += modules.StartModule + enabled += modules.ActorStartModule + enabled += modules.StorageModule +} + +play.filters { + hosts { + # Allow requests to example.com, its subdomains, and localhost:9000. + allowed = ["localhost:9000","."] + } + enabled += filters.AccessLogFilter + enabled += filters.CustomGzipFilter + enabled += filters.ResponseFilter + disabled += play.filters.csrf.CSRFFilter +} +# Assessment Aggregator Configuration +assessment_direct_aggregation_enabled=true +kafka_assessment_topic="sunbird.assessment.raw" +kafka_topics_contentstate_invalid="sunbird.contentstate.invalid" +assessment_skip_missing_records=true + +# Redis optional support — master switch +# When false (default), Redis is not required. All Redis use cases are either skipped or fall back to in-memory alternatives. +redis.enabled = false +# Viewer module — Cassandra configuration +sunbird_course_keyspace = "sunbird_courses" +assessment_aggregator_table = "assessment_aggregator" +user_activity_agg_table = "user_activity_agg" + +# Feature flags +viewer_enabled = true +assessment_direct_aggregation_enabled = true +user_activity_agg_enabled = true + +# Viewer dispatcher (for aggregator pool) +pekko.actor.viewer-dispatcher { + type = Dispatcher + executor = "thread-pool-executor" + thread-pool-executor { + core-pool-size-min = 8 + core-pool-size-max = 64 + max-pool-size-max = 128 + queue-size = -1 + } + throughput = 1 +} diff --git a/modules/viewer/service/conf/logback-test.xml b/modules/viewer/service/conf/logback-test.xml new file mode 100644 index 00000000..adfa02c6 --- /dev/null +++ b/modules/viewer/service/conf/logback-test.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/modules/viewer/service/conf/logback.xml b/modules/viewer/service/conf/logback.xml new file mode 100644 index 00000000..9f047abf --- /dev/null +++ b/modules/viewer/service/conf/logback.xml @@ -0,0 +1,80 @@ + + + + + + + + + + %d %msg%n + + + + + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSX + Etc/UTC + + timestamp + msg + lname + tname + [ignore] + [ignore] + + + + + + + + + + timestamp + msg + lname + tname + [ignore] + [ignore] + exception + + + 30 + 2048 + sun\.reflect\..*\.invoke.* + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/viewer/service/conf/routes b/modules/viewer/service/conf/routes new file mode 100644 index 00000000..9a5ecd98 --- /dev/null +++ b/modules/viewer/service/conf/routes @@ -0,0 +1,25 @@ +# Routes — Viewer Service +# This file defines all application routes (Higher priority routes first) +OPTIONS /*all @controllers.viewer.ViewController.preflight(all) + +# Health Check +GET /health @controllers.viewer.ViewController.health(request: play.mvc.Http.Request) + +# Granular view lifecycle APIs +POST /v1/view/start @controllers.viewer.ViewController.viewStart(request: play.mvc.Http.Request) +POST /v1/view/update @controllers.viewer.ViewController.viewUpdate(request: play.mvc.Http.Request) +POST /v1/view/end @controllers.viewer.ViewController.viewEnd(request: play.mvc.Http.Request) + +# Assessment lifecycle (api.view.assess / api.assessment.read) +POST /v1/assessment/submit @controllers.viewer.ViewController.assessmentSubmit(request: play.mvc.Http.Request) +POST /v1/assessment/read @controllers.viewer.ViewController.assessmentRead(request: play.mvc.Http.Request) + +# Recompute roll-up (admin/ops) — mirrors legacy /v1/activity/agg, targets the aggregator +POST /v1/view/agg @controllers.viewer.ViewAggregateController.agg(request: play.mvc.Http.Request) + +# Read + summary APIs +POST /v1/view/read @controllers.viewer.ViewController.viewRead(request: play.mvc.Http.Request) +POST /v1/summary/read @controllers.viewer.ViewSummaryController.summaryRead(request: play.mvc.Http.Request) +GET /v1/summary/list/:userId @controllers.viewer.ViewSummaryController.summaryList(userId: String, request: play.mvc.Http.Request) +GET /v1/summary/download/:userId @controllers.viewer.ViewSummaryController.summaryDownload(userId: String, request: play.mvc.Http.Request) +DELETE /v1/summary/delete/:userId @controllers.viewer.ViewSummaryController.summaryDelete(userId: String, request: play.mvc.Http.Request) diff --git a/modules/viewer/service/pom.xml b/modules/viewer/service/pom.xml new file mode 100644 index 00000000..7a1a4314 --- /dev/null +++ b/modules/viewer/service/pom.xml @@ -0,0 +1,654 @@ + + + + + org.sunbird + viewer + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + viewer-service-impl + play2 + Viewer Service Module + Viewer Service - Play Framework Application + + + + scalaz-bintray + Scalaz Bintray - releases + https://dl.bintray.com/scalaz/releases/ + + false + + + + + + + typesafe-releases-plugins + https://repo.typesafe.com/typesafe/releases/ + + false + + + + + + + + + com.google.guava + guava + ${guava.version} + + + + + com.google.inject + guice + ${guice.version} + + + com.google.inject.extensions + guice-assistedinject + ${guice.version} + + + com.google.inject.extensions + guice-servlet + ${guice.version} + + + + + org.ow2.asm + asm + ${asm.version} + + + org.ow2.asm + asm-commons + ${asm.version} + + + org.ow2.asm + asm-tree + ${asm.version} + + + org.ow2.asm + asm-util + ${asm.version} + + + + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.module + jackson-module-scala_${scala.major.version} + ${jackson.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + ${jackson.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + ${jackson.version} + + + com.fasterxml.jackson.module + jackson-module-parameter-names + ${jackson.version} + + + + + org.playframework + play_${scala.major.version} + ${play2.version} + + + com.fasterxml.jackson.core + jackson-core + + + joda-time + joda-time + + + com.google.guava + guava + + + org.lz4 + lz4-java + + + org.lz4 + lz4-java + + + at.yawk.lz4 + lz4-java + + + + + org.playframework + play-guice_${scala.major.version} + ${play2.version} + + + com.google.inject + guice + + + com.google.inject.extensions + guice-assistedinject + + + + + org.playframework + play-filters-helpers_${scala.major.version} + ${play2.version} + + + org.playframework + play-netty-server_${scala.major.version} + ${play2.version} + runtime + + + io.netty + netty-codec-http + + + + + org.playframework + play-pekko-http-server_${scala.major.version} + ${play2.version} + + + + + com.google.inject + guice + ${guice.version} + + + com.google.inject.extensions + guice-assistedinject + ${guice.version} + + + + + io.netty + netty-common + ${netty.version} + + + io.netty + netty-buffer + ${netty.version} + + + io.netty + netty-transport + ${netty.version} + + + io.netty + netty-codec + ${netty.version} + + + io.netty + netty-handler + ${netty.version} + + + io.netty + netty-resolver + ${netty.version} + + + io.netty + netty-transport-native-unix-common + ${netty.version} + + + io.netty + netty-transport-native-epoll + ${netty.version} + + + io.netty + netty-transport-classes-epoll + ${netty.version} + + + io.netty + netty-codec-dns + ${netty.version} + + + io.netty + netty-resolver-dns + ${netty.version} + + + io.netty + netty-codec-http + ${netty.version} + + + + + org.sunbird + viewer-actors + 1.0-SNAPSHOT + + + org.sunbird + course-actors-common + 1.0-SNAPSHOT + + + cassandra-unit + org.cassandraunit + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + org.ow2.asm + asm + + + org.ow2.asm + asm-commons + + + org.ow2.asm + asm-analysis + + + org.ow2.asm + asm-tree + + + org.ow2.asm + asm-util + + + com.google.inject.extensions + guice-multibindings + + + + + org.sunbird + course-actors + 1.0-SNAPSHOT + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + com.google.inject.extensions + guice-multibindings + + + + + org.sunbird + enrolment-actor + 1.0-SNAPSHOT + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + com.google.inject.extensions + guice-multibindings + + + + + org.sunbird + activity-aggregator + 1.0-SNAPSHOT + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + com.google.inject.extensions + guice-multibindings + + + + + org.sunbird + actor-util + 1.0-SNAPSHOT + + + org.sunbird + sunbird-actor-utils + 1.0-SNAPSHOT + + + org.sunbird + sunbird-platform-common + 1.0-SNAPSHOT + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + org.apache.logging.log4j + log4j-api + + + + + org.sunbird + assessment-aggregator + 1.0-SNAPSHOT + + + org.lz4 + lz4-java + + + at.yawk.lz4 + lz4-java + + + + + + + com.typesafe + config + 1.3.0 + + + org.scala-lang + scala-library + ${scala.version} + + + com.github.danielwegener + logback-kafka-appender + 0.2.0-RC2 + + + org.lz4 + lz4-java + + + at.yawk.lz4 + lz4-java + + + + + net.logstash.logback + logstash-logback-encoder + ${logstash-logback-encoder.version} + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + + + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-api + 2.17.1 + + + org.apache.logging.log4j + log4j-core + 2.17.1 + + + org.apache.logging.log4j + log4j-to-slf4j + 2.17.1 + + + + + org.ow2.asm + asm + ${asm.version} + + + org.ow2.asm + asm-commons + ${asm.version} + + + org.ow2.asm + asm-analysis + ${asm.version} + + + org.ow2.asm + asm-tree + ${asm.version} + + + org.ow2.asm + asm-util + ${asm.version} + + + + + org.playframework + play-specs2_${scala.major.version} + ${play2.version} + test + + + com.google.code.gson + gson + + + com.google.guava + guava + + + byte-buddy + net.bytebuddy + + + + + org.apache.pekko + pekko-testkit_${scala.major.version} + ${pekko.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + junit + junit + 4.13.1 + test + + + + + viewer-service-${project.version} + ${basedir}/app + ${basedir}/test + + + + ${basedir}/conf + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + + + com.google.code.play2-maven-plugin + play2-maven-plugin + ${play2.plugin.version} + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0 + + false + @{argLine} -Xmx1024m + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.text=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + --add-opens java.base/java.nio.charset=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/sun.net.www.protocol.jar=ALL-UNNAMED + + **/*Spec.java + **/*Test.java + + + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco-maven-plugin.version} + + ${project.build.directory}/jacoco.exec + ${project.build.directory}/jacoco.exec + + + + jacoco-initialize + + prepare-agent + + + + jacoco-site + package + + report + + + + + + + diff --git a/pom.xml b/pom.xml index bb7a4566..e7117252 100644 --- a/pom.xml +++ b/pom.xml @@ -98,12 +98,22 @@ + + + viewer + + modules/lms + modules/viewer + + + lern modules/userorg modules/lms + modules/viewer modules/notification modules/lern/service lern-jacoco-report From 49e48262cf38d95412c8f384b0a31b719fb18909 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Fri, 31 Jul 2026 16:02:55 +0530 Subject: [PATCH 02/30] chore(viewer): keep migration cql local-only Untrack viewer.cql (prod migration) and viewer-test-keyspace.cql and gitignore them; they remain on disk but are not versioned. --- .gitignore | 5 +- .../migrations/viewer-test-keyspace.cql | 154 ------------------ modules/viewer/migrations/viewer.cql | 78 --------- 3 files changed, 4 insertions(+), 233 deletions(-) delete mode 100644 modules/viewer/migrations/viewer-test-keyspace.cql delete mode 100644 modules/viewer/migrations/viewer.cql diff --git a/.gitignore b/.gitignore index ed7e5c39..a0465425 100644 --- a/.gitignore +++ b/.gitignore @@ -101,4 +101,7 @@ scripts/.keycloak-build/ keys/ # Claude -.claude/ \ No newline at end of file +.claude/ +# viewer migration scripts kept local-only +modules/viewer/migrations/viewer.cql +modules/viewer/migrations/viewer-test-keyspace.cql diff --git a/modules/viewer/migrations/viewer-test-keyspace.cql b/modules/viewer/migrations/viewer-test-keyspace.cql deleted file mode 100644 index 054f3394..00000000 --- a/modules/viewer/migrations/viewer-test-keyspace.cql +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Viewer test keyspace — CREATE-from-scratch (NOT the prod migration). - * - * Purpose: spin up an isolated keyspace with every table the viewer touches, already in the - * generalised (viewer_enabled=true) shape — collectionid/contextid, optional_nodes, user_skills, - * assessment_aggregator on collection_id/context_id. Use it to test the viewer without touching the - * live sunbird_courses. (The prod path is viewer.cql: in-place ALTER ... RENAME on sunbird_courses.) - * - * Usage: - * 1. Pick a keyspace name (default below = sunbird_courses_test). To rename, search/replace - * "sunbird_courses_test" throughout this file. - * 2. Run: cqlsh -f viewer-test-keyspace.cql (or ycqlsh for YugabyteDB) - * 3. Point the service at it: sunbird_course_keyspace=sunbird_courses_test - * (env var or externalresource.properties; read env-first via ProjectUtil.getConfigValue). - * Hierarchy tables are separate (hierarchy_store_keyspace) and are NOT created here. - * - * Target: YugabyteDB (YCQL). The `WITH transactions = {'enabled':'true'}` clauses are required by - * YCQL for tables that carry secondary indexes. On apache Cassandra, drop those WITH clauses. - */ - -CREATE KEYSPACE IF NOT EXISTS sunbird_courses_test - WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; - -/* --- UDT used by assessment_aggregator.question --- */ -CREATE TYPE IF NOT EXISTS sunbird_courses_test.question ( - id text, - assess_ts timestamp, - max_score double, - score double, - type text, - title text, - resvalues frozen>>>, - params frozen>>>, - description text, - duration decimal -); - -/* --- user_content_consumption: per-content view state (viewer ucc). Identity = collection/context. --- */ -CREATE TABLE IF NOT EXISTS sunbird_courses_test.user_content_consumption ( - userid text, - collectionid text, - contextid text, - contentid text, - completedcount int, - completionpercentage float, - datetime timestamp, - last_access_time timestamp, - last_completed_time timestamp, - last_updated_time timestamp, - lastaccesstime text, - lastcompletedtime text, - lastupdatedtime text, - progress int, - progressdetails text, - status int, - viewcount int, - PRIMARY KEY (userid, collectionid, contextid, contentid) -) WITH CLUSTERING ORDER BY (collectionid ASC, contextid ASC, contentid ASC); - -/* --- user_enrolments: per-enrolment progress + per-learner optionality (optional_nodes). --- */ -CREATE TABLE IF NOT EXISTS sunbird_courses_test.user_enrolments ( - userid text, - collectionid text, - contextid text, - active boolean, - addedby text, - certificates list>>, - certstatus int, - completedon timestamp, - completionpercentage int, - contentstatus map, - datetime timestamp, - enrolled_date timestamp, - enrolleddate text, - issued_certificates list>>, - lastcontentaccesstime timestamp, - lastreadcontentid text, - lastreadcontentstatus int, - progress int, - status int, - optional_nodes set, - PRIMARY KEY (userid, collectionid, contextid) -) WITH CLUSTERING ORDER BY (collectionid ASC, contextid ASC) - AND transactions = {'enabled': 'true'}; - -CREATE INDEX IF NOT EXISTS user_enrolments_by_collection ON sunbird_courses_test.user_enrolments (collectionid, userid, contextid) - INCLUDE (status, completionpercentage, enrolled_date, datetime); - -/* --- course_batch: a batch belongs to a collection (root batch or chained rootBatch:courseId child). --- */ -CREATE TABLE IF NOT EXISTS sunbird_courses_test.course_batch ( - collectionid text, - contextid text, - cert_templates map>>, - created_date timestamp, - createdby text, - createddate text, - createdfor list, - description text, - end_date timestamp, - enddate text, - enrollment_enddate timestamp, - enrollmentenddate text, - enrollmenttype text, - mentors list, - name text, - start_date timestamp, - startdate text, - status int, - tandc boolean, - updated_date timestamp, - updateddate text, - PRIMARY KEY (collectionid, contextid) -) WITH CLUSTERING ORDER BY (contextid ASC); - -/* --- user_activity_agg: per-node rollup aggregate (activity_id = node do-id, context_id = "cb:"+contextid). --- */ -CREATE TABLE IF NOT EXISTS sunbird_courses_test.user_activity_agg ( - activity_type text, - activity_id text, - user_id text, - context_id text, - agg map, - agg_details list, - agg_last_updated map, - aggregates map, - PRIMARY KEY ((activity_type, activity_id, user_id), context_id) -) WITH CLUSTERING ORDER BY (context_id ASC); - -/* --- assessment_aggregator: per-attempt scores. Identity generalised to collection_id/context_id. --- */ -CREATE TABLE IF NOT EXISTS sunbird_courses_test.assessment_aggregator ( - collection_id text, - context_id text, - user_id text, - content_id text, - attempt_id text, - created_on timestamp, - grand_total text, - last_attempted_on timestamp, - question list>, - total_max_score double, - total_score double, - updated_on timestamp, - PRIMARY KEY (collection_id, context_id, user_id, content_id, attempt_id) -) WITH CLUSTERING ORDER BY (context_id ASC, user_id ASC, content_id ASC, attempt_id ASC) - AND transactions = {'enabled': 'true'}; - --- getUserAssessments filters by user_id (not the partition key) -> needs this index. -CREATE INDEX IF NOT EXISTS assessment_aggregator_by_user ON sunbird_courses_test.assessment_aggregator (user_id, collection_id, context_id, content_id, attempt_id) - INCLUDE (total_score, total_max_score, last_attempted_on); - -/* --- user_skills: durable achieved-skill set, credited once at LP completion (design §6). --- */ -CREATE TABLE IF NOT EXISTS sunbird_courses_test.user_skills ( - userid text PRIMARY KEY, - skills set -); diff --git a/modules/viewer/migrations/viewer.cql b/modules/viewer/migrations/viewer.cql deleted file mode 100644 index d0082145..00000000 --- a/modules/viewer/migrations/viewer.cql +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Viewer module — generalise the course tables to COLLECTION tables. - * Keyspace: sunbird_courses. Authoritative live schema: - * sunbird-spark-installer/scripts/sunbird-yugabyte-migrations/sunbird-lern/sunbird_courses.cql - * - * DECISION: generalise for collection (not course). Rename the identifying columns to - * collectionid / contextid, and add per-learner optionality. - * - * WHY RENAME (not new table / not backfill): - * - courseid / batchid are PRIMARY-KEY columns; Cassandra/Yugabyte RENAME of PK columns is - * METADATA-ONLY: instant, no data copy, no backfill, no PK restructure (same key, new names). - * - Existing data carries over correctly: old courseid -> collectionid, old batchid -> contextid. - * A course IS a collection, so legacy rows are already right (collectionid = the course do-id, - * contextid = the batch). No row-level migration. - * - * COST (code, not data) — must ship in lockstep: - * - Every reader/writer of courseid/batchid switches to collectionid/contextid: - * ActivityAggregateUtil (createProgressUpdateMap / createContentConsumptionUpdateMap), - * LMS actors, lern-data-pipeline jobs. - * - Secondary index on courseid is dropped + recreated on collectionid (below). - * - * NOTE: verify RENAME support + index handling on the target Yugabyte (YCQL) version before prod. - */ - -/* --- user_content_consumption: generalise identity to collection --- */ -ALTER TABLE sunbird_courses.user_content_consumption RENAME courseid TO collectionid; -ALTER TABLE sunbird_courses.user_content_consumption RENAME batchid TO contextid; - -/* --- user_enrolments: generalise identity + add per-learner optionality --- - * VERIFIED on YCQL (cossdev): a column used in an index CANNOT be renamed - * ("Feature Not Yet Implemented. Can't rename column used in an index"). - * So DROP the index BEFORE renaming, then recreate it on the new name. Order matters. */ -DROP INDEX IF EXISTS sunbird_courses.user_enrolments_by_course; -ALTER TABLE sunbird_courses.user_enrolments RENAME courseid TO collectionid; -ALTER TABLE sunbird_courses.user_enrolments RENAME batchid TO contextid; -ALTER TABLE sunbird_courses.user_enrolments ADD optional_nodes set; -- per-enrolment optional child/leaf ids (null = strict) --- recreate with the SAME shape as the live index (composite + INCLUDE), just on the new column name -CREATE INDEX IF NOT EXISTS user_enrolments_by_collection ON sunbird_courses.user_enrolments (collectionid, userid, contextid) - INCLUDE (status, completionpercentage, enrolled_date, datetime); -- index build scans the table; run off-peak on large data - -/* --- course_batch: generalise identity (batch belongs to a collection); no index -> rename directly --- */ -ALTER TABLE sunbird_courses.course_batch RENAME courseid TO collectionid; -ALTER TABLE sunbird_courses.course_batch RENAME batchid TO contextid; - -/* --- assessment_aggregator: generalise identity (viewer now OWNS the assessment path) --- - * PK ((course_id, batch_id), user_id, content_id, attempt_id). course_id/batch_id are PARTITION-KEY - * columns -> RENAME is metadata-only (same as the others). The standard by_user index is on user_id - * and is unaffected; IF any index references course_id/batch_id, DROP it first then recreate on the - * new name (see the user_enrolments pattern above). NOTE snake_case names here (matches this table). */ -DROP INDEX IF EXISTS sunbird_courses.assessment_aggregator_by_user; -- only if it references course_id/batch_id; harmless otherwise -ALTER TABLE sunbird_courses.assessment_aggregator RENAME course_id TO collection_id; -ALTER TABLE sunbird_courses.assessment_aggregator RENAME batch_id TO context_id; --- recreate the by_user index if you dropped it (adjust columns to your live definition): --- CREATE INDEX IF NOT EXISTS assessment_aggregator_by_user ON sunbird_courses.assessment_aggregator (user_id); - -/* - * Reference — post-rename live tables the viewer uses (do NOT recreate): - * - * course_batch PK (collectionid, contextid) - * user_enrolments PK (userid, collectionid, contextid) -- + optional_nodes - * used by viewer: progress, status, completionpercentage, contentstatus, completedon, lastread* - * user_content_consumption PK (userid, collectionid, contextid, contentid) - * used by viewer: status, progress, progressdetails, completedcount, viewcount, last_*_time - * user_activity_agg -- unchanged; aggregate identified by collectionId (activity_id = collectionId) - * per user + context_id = "cb:"+contextid - * - * assessment_aggregator: PK ((collection_id, context_id), user_id, content_id, attempt_id) -- renamed above - * used by viewer: total_score, total_max_score, grand_total, question, created_on, last_attempted_on - */ - -/* - * LP durable skill store (design §6). Per-user achieved-skill set, credited ONCE at LP completion. - * NOTE: distinct from the legacy `sunbird.user_skills` endorsement table (different keyspace) — no collision. - */ -CREATE TABLE IF NOT EXISTS sunbird_courses.user_skills ( - userid text PRIMARY KEY, - skills set -); From d06688d3913b5663618423eeb475063765eced8f Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Fri, 31 Jul 2026 16:03:05 +0530 Subject: [PATCH 03/30] refactor(viewer): remove eager enrol fan-out; trim unused helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CourseEnrolmentActor: drop enrolTrackableDescendants/readCollectionHierarchy/ collectTrackable (the eager pre-enrol the level-gated walker replaces) + now-unused isViewerEnabled/cassandraOperation/jsonMapper fields. Root enrol only; systemEnroll (the LP auto-enrol seam) kept. - ProgressionPolicy: remove unused hasNestedTrackable/isAssessment/computeAchievedSkills (+ orphaned privates); keep levelOf/coursesOfLevel/orderedLevels/computeOptionalNodes. - ViewerRequestKeys: canonical collectionId/contextId/contentId only (no legacy courseId/batchId fallback — callers resolve). Fix triggerAggregation to send contextId. - ContentConsumptionActor: drop redundant batchId key (viewer reads contextId). --- .../enrolments/ContentConsumptionActor.scala | 1 - .../enrolments/CourseEnrolmentActor.scala | 62 ------------------- .../viewer/actor/ViewConsumptionActor.scala | 2 +- .../viewer/actor/ViewerRequestKeys.scala | 28 +++------ .../viewer/util/ProgressionPolicy.scala | 57 ++--------------- .../viewer/util/ProgressionPolicySpec.scala | 45 -------------- 6 files changed, 13 insertions(+), 182 deletions(-) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala index 69ad0d65..d9e5114d 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala @@ -514,7 +514,6 @@ class ContentConsumptionActor @Inject() ( put("contentId", contentId) put("collectionId", collectionId) put("contextId", c.get(JsonKey.BATCH_ID)) - put(JsonKey.BATCH_ID, c.get(JsonKey.BATCH_ID)) put(JsonKey.USER_ID, userId) Option(c.get("progressdetails")).orElse(Option(c.get("progressDetails"))).foreach(pd => put("progressDetails", pd)) }} diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index 11f2b7f5..dcfd5f12 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -44,9 +44,6 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c var courseBatchDao: CourseBatchDao = new CourseBatchDaoImpl() var userCoursesDao: UserCoursesDao = new UserCoursesDaoImpl() var groupDao: GroupDaoImpl = new GroupDaoImpl() - private lazy val cassandraOperation = org.sunbird.helper.ServiceFactory.getInstance - private val jsonMapper = new ObjectMapper() - private def isViewerEnabled: Boolean = java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) private val redisEnabled: Boolean = RedisCacheUtil.isRedisEnabled val isCacheEnabled = redisEnabled && (if (StringUtils.isNotBlank(ProjectUtil.getConfigValue("user_enrolments_response_cache_enable"))) (ProjectUtil.getConfigValue("user_enrolments_response_cache_enable")).toBoolean else true) @@ -94,9 +91,6 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c validateEnrolment(batchData, enrolmentData, true) val data: java.util.Map[String, AnyRef] = createUserEnrolmentMap(userId, courseId, batchId, enrolmentData, request.getContext.getOrDefault(JsonKey.REQUEST_ID, "").asInstanceOf[String]) upsertEnrollment(userId, courseId, batchId, data, (null == enrolmentData), request.getRequestContext) - // viewer.enabled: also enrol the trackable descendant nodes (best-effort, never fails the root enrol) - if (isViewerEnabled) - enrolTrackableDescendants(userId, courseId, batchId, request.getContext.getOrDefault(JsonKey.REQUEST_ID, "").asInstanceOf[String], request.getRequestContext) if (isCacheEnabled) { logger.info(request.getRequestContext, "CourseEnrolmentActor :: enroll :: Deleting redis for key " + getCacheKey(userId)) cacheUtil.delete(getCacheKey(userId)) @@ -292,62 +286,6 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c } }} - /** - * viewer.enabled: on root enrol, also create a user_enrolments row for every TRACKABLE descendant - * (trackable.enabled == "Yes") so nested-node progress has an enrolment to land in. Batch ids are - * chained to the nearest trackable ancestor (`parentBatch:nodeId`); the matching course_batch rows - * are created out of band (manual / batch-create API with an explicit batchId). Best-effort: any - * failure is logged, never fails the root enrol. Creates NO batches and skips non-trackable nodes. - * ponytail: reads full hierarchy JSON per enrol, no cache — add a TTL cache if enrol throughput needs it. - */ - private def enrolTrackableDescendants(userId: String, rootId: String, rootBatchId: String, requestedBy: String, ctx: RequestContext): Unit = { - try { - val hierarchy = readCollectionHierarchy(rootId, ctx) - if (hierarchy == null) { logger.info(ctx, s"enrolTrackableDescendants: no hierarchy for $rootId"); return } - val acc = scala.collection.mutable.ListBuffer[(String, String)]() - collectTrackable(hierarchy, rootBatchId, acc) - acc.foreach { case (nodeId, nodeBatch) => - if (null == userCoursesDao.read(ctx, userId, nodeId, nodeBatch)) { - val data = createUserEnrolmentMap(userId, nodeId, nodeBatch, null, requestedBy) - upsertEnrollment(userId, nodeId, nodeBatch, data, true, ctx) - logger.info(ctx, s"enrolTrackableDescendants: enrolled node=$nodeId batch=$nodeBatch user=$userId") - } - } - } catch { - case ex: Exception => logger.error(ctx, s"enrolTrackableDescendants failed root=$rootId user=$userId: ${ex.getMessage}", ex) - } - } - - private def readCollectionHierarchy(rootId: String, ctx: RequestContext): java.util.Map[String, AnyRef] = { - val keyspace = Option(ProjectUtil.getConfigValue("hierarchy_store_keyspace")).filter(StringUtils.isNotBlank).getOrElse("dev_hierarchy_store") - val table = Option(ProjectUtil.getConfigValue("content_hierarchy_table")).filter(StringUtils.isNotBlank).getOrElse("content_hierarchy") - val filters = new java.util.HashMap[String, AnyRef]() {{ put("identifier", rootId) }} - val rows = cassandraOperation.getRecordsByProperties(keyspace, table, filters.asInstanceOf[java.util.Map[String, AnyRef]], ctx) - .getResult.getOrDefault(JsonKey.RESPONSE, new java.util.ArrayList[java.util.Map[String, AnyRef]]) - .asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] - if (rows.isEmpty) return null - val json = rows.get(0).get("hierarchy").asInstanceOf[String] - if (StringUtils.isBlank(json)) null else jsonMapper.readValue(json, classOf[java.util.Map[String, AnyRef]]) - } - - /** Recurse children: a trackable node -> (id, parentBatch:id) and becomes the parent batch for its subtree; non-trackable nodes are transparent structure. */ - private def collectTrackable(node: java.util.Map[String, AnyRef], effParentBatch: String, acc: scala.collection.mutable.ListBuffer[(String, String)]): Unit = { - val children = node.get("children") - if (children == null) return - children.asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]].asScala.foreach { child => - val id = child.get("identifier").asInstanceOf[String] - val trackable = child.get("trackable").asInstanceOf[java.util.Map[String, AnyRef]] - val enabled = trackable != null && "Yes".equalsIgnoreCase(String.valueOf(trackable.get("enabled"))) - if (enabled && StringUtils.isNotBlank(id)) { - val nodeBatch = effParentBatch + ":" + id - acc += ((id, nodeBatch)) - collectTrackable(child, nodeBatch, acc) - } else { - collectTrackable(child, effParentBatch, acc) - } - } - } - def notifyUser(userId: String, batchData: CourseBatch, operationType: String): Unit = { val isNotifyUser = java.lang.Boolean.parseBoolean(PropertiesCache.getInstance().getProperty(JsonKey.SUNBIRD_COURSE_BATCH_NOTIFICATIONS_ENABLED)) if(isNotifyUser){ diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala index d08a1e36..76623ec8 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala @@ -218,7 +218,7 @@ class ViewConsumptionActor @Inject() ( aggRequest.setRequestContext(ctx) aggRequest.put(JsonKey.USER_ID, key.get("userid")) aggRequest.put("collectionId", key.get("collectionid")) - aggRequest.put(JsonKey.BATCH_ID, key.get("contextid")) + aggRequest.put("contextId", key.get("contextid")) // Async, fire-and-forget: the rollup + LP progression run in the background on the aggregator // (per-user serialized). The hot path does not wait for it — the change from before is ask -> tell. viewerAggregatorActor.tell(aggRequest, ActorRef.noSender) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala index 8d75c5ae..6ce1573e 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala @@ -4,28 +4,16 @@ import org.apache.commons.lang3.StringUtils import org.sunbird.request.Request /** - * Backward-compatible request-key resolution for the viewer APIs. - * - * The viewer generalises course -> collection. Clients (and the content-state delegation) may send - * the OLD keys courseId/batchId or the NEW keys collectionId/contextId — both are accepted, mapped - * to the collection/context concept. camelCase (API convention) with lowercase fallbacks. - * This is the REQUEST-payload layer only; DB columns are handled separately. + * Canonical viewer request keys — the viewer contract is collectionId / contextId / contentId only. + * Legacy courseId/batchId resolution is the caller's job (the content-consumption adapter maps them + * before dispatching), so no fallback lives here. Null/blank-safe extraction in one place. */ object ViewerRequestKeys { - private def firstNonBlank(request: Request, keys: String*): Option[String] = - keys.iterator - .map(k => request.get(k)) - .collectFirst { case v: String if StringUtils.isNotBlank(v) => v } + private def value(request: Request, key: String): Option[String] = + Option(request.get(key)).collect { case s: String if StringUtils.isNotBlank(s) => s } - /** collectionId, else legacy courseId. */ - def collectionId(request: Request): Option[String] = - firstNonBlank(request, "collectionId", "collectionid", "courseId", "courseid") - - /** contextId, else legacy batchId. */ - def contextId(request: Request): Option[String] = - firstNonBlank(request, "contextId", "contextid", "batchId", "batchid") - - def contentId(request: Request): String = - firstNonBlank(request, "contentId", "contentid").orNull + def collectionId(request: Request): Option[String] = value(request, "collectionId") + def contextId(request: Request): Option[String] = value(request, "contextId") + def contentId(request: Request): String = value(request, "contentId").orNull } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala index 5d035a0c..1756aa49 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala @@ -1,52 +1,15 @@ package org.sunbird.viewer.util -import scala.jdk.CollectionConverters._ - /** * Pure, host-agnostic decisions for Learning-Path progression. No I/O — every input is passed in, * so these are trivially unit-testable and run identically in-request or in the async aggregator. - * (Structural helpers only for now; level/optionality helpers are added in a later slice.) */ object ProgressionPolicy { - private val ASSESSMENT_CATEGORY = "practice question set" - - private def isTrackable(node: java.util.Map[String, AnyRef]): Boolean = - node.get("trackable") match { - case t: java.util.Map[_, _] => - "Yes".equalsIgnoreCase(String.valueOf(t.asInstanceOf[java.util.Map[String, AnyRef]].get("enabled"))) - case _ => false - } - - private def childrenOf(node: java.util.Map[String, AnyRef]): List[java.util.Map[String, AnyRef]] = - node.get("children") match { - case l: java.util.List[_] => l.asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]].asScala.toList - case _ => Nil - } - - /** - * Structural LP detection: true iff the collection has a descendant that is itself a trackable - * collection (`trackable.enabled == "Yes"`). A plain course — whose children are non-trackable - * content — is false. No reliance on `policy`/`primaryCategory`. - */ - def hasNestedTrackable(rootNode: java.util.Map[String, AnyRef]): Boolean = { - def hasTrackableDescendant(node: java.util.Map[String, AnyRef]): Boolean = - childrenOf(node).exists(c => isTrackable(c) || hasTrackableDescendant(c)) - hasTrackableDescendant(rootNode) - } - - /** A course is an assessment course iff it has a child with `primaryCategory == "Practice Question Set"`. */ - def isAssessment(courseNode: java.util.Map[String, AnyRef]): Boolean = - childrenOf(courseNode).exists(c => - ASSESSMENT_CATEGORY.equalsIgnoreCase(String.valueOf(c.get("primaryCategory")))) - - // ── Level helpers + optionality resolver (pure; take their data as parameters, no I/O) ── - /** - * The course's level = the ancestor that is a **direct child of the root** (top-most under root), - * i.e. `lastOption` of the ancestor chain excluding the root — NOT the nearest ancestor. For a - * 2-deep root->level->course tree they coincide; only this definition is correct if a course is - * nested deeper inside a level. `ancestorsOf` returns the chain nearest-first. + * The course's level = the ancestor that is a direct child of the root (top-most under root), + * i.e. lastOption of the ancestor chain excluding the root — NOT the nearest ancestor. `ancestorsOf` + * returns the chain nearest-first. */ def levelOf(course: String, ancestorsOf: String => List[String], root: String): Option[String] = ancestorsOf(course).filterNot(_ == root).lastOption @@ -63,8 +26,7 @@ object ProgressionPolicy { /** * A course is optional iff it is not an assessment and all of its (non-empty) skills are achieved. - * `Strict` waives nothing. `Adaptive`/`PriorLearning` differ only in how `skillsAchieved` is built - * by the caller — this function is policy-agnostic beyond the `Strict` short-circuit. + * `Strict` waives nothing; other policies differ only in how the caller builds `skillsAchieved`. */ def computeOptionalNodes(policy: String, courses: List[String], skillsByCourse: Map[String, Set[String]], @@ -76,15 +38,4 @@ object ProgressionPolicy { !assessmentCourses.contains(c) && skills.nonEmpty && skills.subsetOf(skillsAchieved) }.toSet } - - /** - * A skill is achieved iff **all** of its tagged questions are correct. Pure core of `skillsFrom` - * (the aggregator supplies `skillToQuestions` from `/v3/search` tags and `correctQuestions` from - * `assessment_aggregator`). Skills with no tagged questions are never achieved. - */ - def computeAchievedSkills(skillToQuestions: Map[String, Set[String]], - correctQuestions: Set[String]): Set[String] = - skillToQuestions.collect { - case (skill, qs) if qs.nonEmpty && qs.subsetOf(correctQuestions) => skill - }.toSet } diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala index e5c249f1..195d8646 100644 --- a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala @@ -3,43 +3,8 @@ package org.sunbird.viewer.util import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import scala.jdk.CollectionConverters._ - class ProgressionPolicySpec extends AnyFlatSpec with Matchers { - private def node(fields: (String, AnyRef)*): java.util.Map[String, AnyRef] = - fields.toMap.asJava - - private def trackable(id: String, children: java.util.Map[String, AnyRef]*): java.util.Map[String, AnyRef] = - node("identifier" -> id, - "trackable" -> node("enabled" -> "Yes"), - "children" -> children.toList.asJava) - - private def child(primaryCategory: String): java.util.Map[String, AnyRef] = - node("primaryCategory" -> primaryCategory) - - "hasNestedTrackable" should "be true for a trackable collection containing a nested trackable collection" in { - val lp = trackable("do_lp", trackable("CRS-A")) // a trackable collection nested inside a trackable collection - ProgressionPolicy.hasNestedTrackable(lp) shouldBe true - } - - it should "be false for a plain course whose children are non-trackable content" in { - val course = trackable("do_course", node("identifier" -> "c1")) // child is not a trackable collection - ProgressionPolicy.hasNestedTrackable(course) shouldBe false - } - - it should "be false when there are no children at all" in { - ProgressionPolicy.hasNestedTrackable(node("identifier" -> "leaf")) shouldBe false - } - - "isAssessment" should "be true when a Practice Question Set child exists" in { - ProgressionPolicy.isAssessment(trackable("CRS", child("Practice Question Set"))) shouldBe true - } - - it should "be false for a content-only course" in { - ProgressionPolicy.isAssessment(trackable("CRS", child("Explanation Content"))) shouldBe false - } - // Reference tree: L1[CRS-A] L2[CRS-B,CRS-C] L3[CRS-D,CRS-E] L4[CRS-F] (ancestors nearest-first) private val anc: Map[String, List[String]] = Map( "CRS-A" -> List("L1", "do_lp"), @@ -80,14 +45,4 @@ class ProgressionPolicySpec extends AnyFlatSpec with Matchers { ProgressionPolicy.computeOptionalNodes("Strict", List("CRS-B"), Map("CRS-B" -> Set("s1")), Set.empty, Set("s1")) shouldBe empty } - - "computeAchievedSkills" should "return skills whose questions are ALL correct" in { - val skillQs = Map("s1" -> Set("q1", "q2"), "s2" -> Set("q3"), "s3" -> Set("q4", "q5")) - val correct = Set("q1", "q2", "q3", "q4") // s1 all correct, s2 all correct, s3 missing q5 - ProgressionPolicy.computeAchievedSkills(skillQs, correct) shouldBe Set("s1", "s2") - } - - it should "ignore skills that have no tagged questions" in { - ProgressionPolicy.computeAchievedSkills(Map("s0" -> Set.empty[String]), Set("q1")) shouldBe empty - } } From a0ba0fbb62ecd5b865efaf508eca923634e4e583 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Fri, 31 Jul 2026 16:27:16 +0530 Subject: [PATCH 04/30] test(activity): consolidate HierarchyRelationsUtil trackable + cache tests into one spec --- ...scala => HierarchyRelationsUtilTest.scala} | 42 +++++++++++++------ .../HierarchyRelationsUtilTrackableTest.scala | 40 ------------------ 2 files changed, 30 insertions(+), 52 deletions(-) rename modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/{HierarchyRelationsUtilCacheTest.scala => HierarchyRelationsUtilTest.scala} (52%) delete mode 100644 modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTrackableTest.scala diff --git a/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilCacheTest.scala b/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTest.scala similarity index 52% rename from modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilCacheTest.scala rename to modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTest.scala index 9cec173e..e041bcb0 100644 --- a/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilCacheTest.scala +++ b/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTest.scala @@ -10,11 +10,10 @@ import org.sunbird.request.RequestContext import org.sunbird.response.Response /** - * Tests the JVM-wide TTL cache added to HierarchyRelationsUtil.readFromDB: a repeated lookup for the - * same relationship_key is served from memory (DB hit once), and empty results are NOT cached (so a - * freshly-published collection is not held stale). Unique keys per test avoid cross-test cache bleed. + * HierarchyRelationsUtil: the trackablenodes reader and the JVM-wide TTL cache on readFromDB. + * Unique relationship keys per test avoid cross-test cache bleed. */ -class HierarchyRelationsUtilCacheTest extends AnyFlatSpec with Matchers with MockFactory { +class HierarchyRelationsUtilTest extends AnyFlatSpec with Matchers with MockFactory { private def responseWith(nodeIds: util.List[String]): Response = { val row = new util.HashMap[String, AnyRef]() {{ put("node_ids", nodeIds) }} @@ -26,15 +25,35 @@ class HierarchyRelationsUtilCacheTest extends AnyFlatSpec with Matchers with Moc val l = new util.ArrayList[String](); ids.foreach(l.add); l } + // --- getTrackableNodes: reads ::trackablenodes, ordered, no dedup --- + + "getTrackableNodes" should "return the ordered trackable ids for the root" in { + val ops = mock[CassandraOperation] + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(responseWith(nodeList("CRS-A", "CRS-B", "CRS-C"))).once() + val u = HierarchyRelationsUtil(ops) + u.getTrackableNodes("trk-root-unique-1", null) shouldBe List("CRS-A", "CRS-B", "CRS-C") + } + + it should "return empty when no trackablenodes relation exists" in { + val ops = mock[CassandraOperation] + val emptyResp = new Response(); emptyResp.put("response", new util.ArrayList[util.Map[String, AnyRef]]()) + (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(emptyResp).once() + val u = HierarchyRelationsUtil(ops) + u.getTrackableNodes("trk-root-unique-2", null) shouldBe empty + } + + // --- readFromDB TTL cache: repeat served from memory; empty NOT cached --- + "getLeafNodes" should "hit the DB once and serve the repeat from cache" in { val ops = mock[CassandraOperation] - // 4-arg getRecordsByProperties is what readFromDB uses; expect EXACTLY one DB call for two lookups (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) .expects(*, *, *, *).returns(responseWith(nodeList("leaf-A", "leaf-B"))).once() - val util0 = HierarchyRelationsUtil(ops) + val u = HierarchyRelationsUtil(ops) val col = "cacheHit-collection-unique-1" - val first = util0.getLeafNodes(col, col, null) - val second = util0.getLeafNodes(col, col, null) + val first = u.getLeafNodes(col, col, null) + val second = u.getLeafNodes(col, col, null) first should contain allOf("leaf-A", "leaf-B") second shouldBe first } @@ -43,14 +62,13 @@ class HierarchyRelationsUtilCacheTest extends AnyFlatSpec with Matchers with Moc val ops = mock[CassandraOperation] val emptyResp = new Response(); emptyResp.put("response", new util.ArrayList[util.Map[String, AnyRef]]()) val populated = responseWith(nodeList("leaf-X")) - // first lookup empty (not published yet), second returns data -> BOTH must hit the DB (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) .expects(*, *, *, *).returns(emptyResp).once() (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) .expects(*, *, *, *).returns(populated).once() - val util0 = HierarchyRelationsUtil(ops) + val u = HierarchyRelationsUtil(ops) val col = "negativeCache-collection-unique-2" - util0.getLeafNodes(col, col, null) shouldBe empty - util0.getLeafNodes(col, col, null) should contain("leaf-X") + u.getLeafNodes(col, col, null) shouldBe empty + u.getLeafNodes(col, col, null) should contain("leaf-X") } } diff --git a/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTrackableTest.scala b/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTrackableTest.scala deleted file mode 100644 index 581242fa..00000000 --- a/modules/lms/activity-aggregator/src/test/scala/org/sunbird/activity/util/HierarchyRelationsUtilTrackableTest.scala +++ /dev/null @@ -1,40 +0,0 @@ -package org.sunbird.activity.util - -import java.util - -import org.scalamock.scalatest.MockFactory -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -import org.sunbird.cassandra.CassandraOperation -import org.sunbird.request.RequestContext -import org.sunbird.response.Response - -/** getTrackableNodes reads the `::trackablenodes` relation, ordered, without dedup. */ -class HierarchyRelationsUtilTrackableTest extends AnyFlatSpec with Matchers with MockFactory { - - private def responseWith(nodeIds: util.List[String]): Response = { - val row = new util.HashMap[String, AnyRef]() {{ put("node_ids", nodeIds) }} - val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(row) - val r = new Response(); r.put("response", rows); r - } - private def nodeList(ids: String*): util.List[String] = { - val l = new util.ArrayList[String](); ids.foreach(l.add); l - } - - "getTrackableNodes" should "return the ordered trackable ids for the root" in { - val ops = mock[CassandraOperation] - (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) - .expects(*, *, *, *).returns(responseWith(nodeList("CRS-A", "CRS-B", "CRS-C"))).once() - val u = HierarchyRelationsUtil(ops) - u.getTrackableNodes("trk-root-unique-1", null) shouldBe List("CRS-A", "CRS-B", "CRS-C") - } - - it should "return empty when no trackablenodes relation exists" in { - val ops = mock[CassandraOperation] - val emptyResp = new Response(); emptyResp.put("response", new util.ArrayList[util.Map[String, AnyRef]]()) - (ops.getRecordsByProperties(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) - .expects(*, *, *, *).returns(emptyResp).once() - val u = HierarchyRelationsUtil(ops) - u.getTrackableNodes("trk-root-unique-2", null) shouldBe empty - } -} From 3e1ec7ce49c265091210ddd28b806f5f0c1bc79d Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Fri, 31 Jul 2026 16:27:43 +0530 Subject: [PATCH 05/30] docs(viewer): drop stale ref to removed ProgressionPolicy.computeAchievedSkills --- .../scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index c0552a38..7f04a052 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -189,7 +189,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // VERIFY-ON-DEPLOY: per-course skills (se_skills) + assessment flag via /v3/search. private def courseMeta(trackable: List[String], ctx: RequestContext): Map[String, (Set[String], Boolean)] = trackable.map(c => c -> (Set.empty[String], isAssessmentCourse(c, ctx))).toMap - // VERIFY-ON-DEPLOY: best-attempt assessment_aggregator scores × se_skills tags -> ProgressionPolicy.computeAchievedSkills. + // VERIFY-ON-DEPLOY: derive from best-attempt assessment_aggregator scores × se_skills tags (skill achieved = all its questions correct). private def skillsFromAssessment(userId: String, rootId: String, courseId: String, ctx: RequestContext): Set[String] = Set.empty private def isComplete(userId: String, courseId: String, rootBatchId: String, ctx: RequestContext): Boolean = From dfda98c2f93ac7532222532043ee350ee9b1240f Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Fri, 31 Jul 2026 16:48:09 +0530 Subject: [PATCH 06/30] ci(viewer): build + deploy standalone viewer-service image Enable the viewer to run as its own deployable service (deployment_mode=distributed) alongside the monolith path (viewer-actors in lern-service, unchanged): - build/viewer/Dockerfile (Play dist viewer-service-1.0-SNAPSHOT). - build-local.sh / docker-build.sh: add 'viewer' service (profile=viewer, module=modules/viewer/service, dist=viewer-service-1.0-SNAPSHOT-dist.zip). - deploy.yml: gated Build/Push Viewer Service steps (VIEWER_SERVICE_BUILD var) -> lern-viewer-service image. - pr-checks.yml: viewer-build job (compile+test both viewer submodules) wired into sonar aggregation. Verified: mvn -P viewer reactor (19 modules) + play2:dist produce the dist zip. --- .github/workflows/deploy.yml | 21 +++++++++++++ .github/workflows/pr-checks.yml | 55 ++++++++++++++++++++++++++++++++- build/viewer/Dockerfile | 37 ++++++++++++++++++++++ scripts/build-local.sh | 5 +-- scripts/docker-build.sh | 7 +++-- 5 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 build/viewer/Dockerfile diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2ab41456..551bacf1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,6 +50,7 @@ jobs: echo "USERORG_ENABLE=${{ vars.USERORG_SERVICE_BUILD == 'true' }}" >> $GITHUB_ENV echo "LMS_ENABLE=${{ vars.LMS_SERVICE_BUILD == 'true' }}" >> $GITHUB_ENV echo "NOTIFICATION_ENABLE=${{ vars.NOTIFICATION_SERVICE_BUILD == 'true' }}" >> $GITHUB_ENV + echo "VIEWER_ENABLE=${{ vars.VIEWER_SERVICE_BUILD == 'true' }}" >> $GITHUB_ENV # Cloud Storage Provider (CSP) — lookup from repo settings, default to azure CSP="${{ vars.CLOUD_STORAGE_PROVIDER }}" @@ -147,3 +148,23 @@ jobs: tags: ${{ env.ORG_BASE }}/lern-notification-service:${{ env.IMAGE_TAG }} build-args: | CSP=${{ env.CSP }} + + # ---------------------------------------------------------------- + # VIEWER SERVICE (standalone / distributed deployment) + # ---------------------------------------------------------------- + - name: Build Viewer Service + if: env.VIEWER_ENABLE == 'true' + env: + CSP: ${{ env.CSP }} + run: ./scripts/build-local.sh --service viewer + + - name: Push Viewer Service Docker + if: env.VIEWER_ENABLE == 'true' + uses: docker/build-push-action@v4 + with: + context: . + file: build/viewer/Dockerfile + push: true + tags: ${{ env.ORG_BASE }}/lern-viewer-service:${{ env.IMAGE_TAG }} + build-args: | + CSP=${{ env.CSP }} diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 34f0a23a..c4a96712 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -201,8 +201,54 @@ jobs: name: lern-artifacts path: modules/lern/service/**/target/** + viewer-build: + needs: build-core + runs-on: ubuntu-latest + + env: + CSP: ${{ vars.CLOUD_STORAGE_PROVIDER || 'azure' }} + + services: + redis: + image: redis:4.0.0 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Restore Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ github.run_id }} + + - name: Build and Generate Coverage Report (Viewer) + run: | + # -am builds the viewer's deps (core + lms modules); viewer-service is the standalone Play app. + mvn install -P viewer,${CSP} -pl modules/viewer/actors,modules/viewer/service -am -DskipTests -Dcheckstyle.skip + mvn verify -P viewer,${CSP} -pl modules/viewer/actors,modules/viewer/service \ + -DreuseForks=false \ + -DargLine="--add-opens java.base/sun.misc=ALL-UNNAMED -Djdk.attach.allowAttachSelf=true" + + - name: Upload Viewer Artifacts + uses: actions/upload-artifact@v4 + with: + name: viewer-artifacts + path: modules/viewer/**/target/** + sonar-analysis: - needs: [userorg-build, lms-build, notification-build, lern-build] + needs: [userorg-build, lms-build, notification-build, lern-build, viewer-build] runs-on: ubuntu-latest env: @@ -252,6 +298,13 @@ jobs: path: . merge-multiple: true + - name: Download Viewer Artifacts + uses: actions/download-artifact@v4 + with: + name: viewer-artifacts + path: . + merge-multiple: true + - name: Run Aggregated SonarQube Analysis env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/build/viewer/Dockerfile b/build/viewer/Dockerfile new file mode 100644 index 00000000..000c8d2d --- /dev/null +++ b/build/viewer/Dockerfile @@ -0,0 +1,37 @@ +# Sunbird Viewer Service Dockerfile (standalone / distributed deployment) +# Stage 1: Extraction +FROM alpine:3.20 AS builder +RUN apk update && apk add unzip +WORKDIR /app +COPY modules/viewer/service/target/viewer-service-1.0-SNAPSHOT-dist.zip . +RUN unzip viewer-service-1.0-SNAPSHOT-dist.zip + +# Stage 2: Runtime +FROM eclipse-temurin:11-jre-alpine + +# Cloud Storage Provider selection — pass --build-arg CSP= at build time. +# Defaults to azure. The Maven build must have been run with the matching profile (-P ). +# cloud_storage_auth_type defaults to OIDC (Kubernetes Workload Identity). +# Override at runtime via K8s ConfigMap or: docker run -e sunbird_cloud_storage_auth_type=ACCESS_KEY +ARG CSP=azure +ENV sunbird_cloud_service_provider=${CSP} +ENV sunbird_cloud_storage_auth_type=OIDC + +RUN apk upgrade --no-cache \ + && apk add --no-cache curl "zlib>=1.3.2-r0" \ + && adduser -u 1001 -h /home/sunbird/ -D sunbird \ + && mkdir -p /home/sunbird/ + +WORKDIR /home/sunbird/ +COPY --from=builder --chown=sunbird:sunbird /app/viewer-service-1.0-SNAPSHOT /home/sunbird/viewer-service-1.0-SNAPSHOT +COPY modules/viewer/service/conf/logback.xml /home/sunbird/viewer-service-1.0-SNAPSHOT/conf/logback.xml + +USER sunbird +EXPOSE 9000 + +CMD java -XX:+PrintFlagsFinal $JAVA_OPTIONS \ + -Dlog4j2.formatMsgNoLookups=true \ + -Dplay.server.http.idleTimeout=180s \ + -Dlogback.configurationFile=/home/sunbird/viewer-service-1.0-SNAPSHOT/conf/logback.xml \ + -cp '/home/sunbird/viewer-service-1.0-SNAPSHOT/lib/viewer-service-1.0-SNAPSHOT.jar:/home/sunbird/viewer-service-1.0-SNAPSHOT/lib/*' \ + play.core.server.ProdServerStart /home/sunbird/viewer-service-1.0-SNAPSHOT diff --git a/scripts/build-local.sh b/scripts/build-local.sh index 36d5a1ca..850cc837 100755 --- a/scripts/build-local.sh +++ b/scripts/build-local.sh @@ -39,6 +39,7 @@ declare -A SERVICE_CONFIG=( [userorg]="userorg|modules/userorg/controller|userorg-service-1.0-SNAPSHOT-dist.zip" [lms]="lms|modules/lms/service|lms-service-1.0-SNAPSHOT-dist.zip" [notification]="notification|modules/notification/service|notification-service-1.0-SNAPSHOT-dist.zip" + [viewer]="viewer|modules/viewer/service|viewer-service-1.0-SNAPSHOT-dist.zip" ) # Help message @@ -46,7 +47,7 @@ function show_help { echo "Usage: $0 --service [options]" echo "" echo "Required:" - echo " -s, --service Service name: lern, userorg, lms, notification" + echo " -s, --service Service name: lern, userorg, lms, notification, viewer" echo "" echo "Options:" echo " -c, --csp Cloud Storage Provider (default: azure)" @@ -82,7 +83,7 @@ fi # Validate service name if [ -z "${SERVICE_CONFIG[$SERVICE]}" ]; then echo "Error: Unknown service '$SERVICE'" - echo "Valid services: lern, userorg, lms, notification" + echo "Valid services: lern, userorg, lms, notification, viewer" exit 1 fi diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 917e4e62..96090b86 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -17,7 +17,7 @@ # ./scripts/docker-build.sh --service notification --repo ghcr.io/myorg --push # # Options: -# -s, --service Service name (required): lern, userorg, lms, notification +# -s, --service Service name (required): lern, userorg, lms, notification, viewer # -r, --repo Docker registry/repository (optional, no push if omitted) # -n, --name Image name (default: service-service) # -t, --tag Image tag (default: latest) @@ -45,6 +45,7 @@ declare -A SERVICE_CONFIG=( [userorg]="userorg-service|build/userorg/Dockerfile|modules/userorg/controller/target/userorg-service-1.0-SNAPSHOT-dist.zip" [lms]="lms-service|build/lms/Dockerfile|modules/lms/service/target/lms-service-1.0-SNAPSHOT-dist.zip" [notification]="notification-service|build/notification/Dockerfile|modules/notification/service/target/notification-service-1.0-SNAPSHOT-dist.zip" + [viewer]="viewer-service|build/viewer/Dockerfile|modules/viewer/service/target/viewer-service-1.0-SNAPSHOT-dist.zip" ) # Help message @@ -52,7 +53,7 @@ function show_help { echo "Usage: $0 --service [options]" echo "" echo "Required:" - echo " -s, --service Service name: lern, userorg, lms, notification" + echo " -s, --service Service name: lern, userorg, lms, notification, viewer" echo "" echo "Options:" echo " -r, --repo Docker registry/repository (e.g., ghcr.io/myorg)" @@ -95,7 +96,7 @@ fi # Validate service name if [ -z "${SERVICE_CONFIG[$SERVICE]}" ]; then echo "Error: Unknown service '$SERVICE'" - echo "Valid services: lern, userorg, lms, notification" + echo "Valid services: lern, userorg, lms, notification, viewer" exit 1 fi From b40552a68001b0e2a9c8dd22f70dad471d325d6e Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Mon, 3 Aug 2026 11:40:12 +0530 Subject: [PATCH 07/30] fix(enrolment): address course_batch/user_enrolments by collectionid/contextid under viewer_enabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gated remap (Util.toCollectionColumns — no-op when viewer disabled) so the legacy enrol path works against the generalised (viewer) schema. Applied in CourseBatchDaoImpl (all methods), UserCoursesDaoImpl (read/update/updateV2/listEnrolments/getBatchParticipants), and CourseEnrolmentActor.upsertEnrollment (insert row). --- .../dao/impl/CourseBatchDaoImpl.java | 9 ++++++-- .../dao/impl/UserCoursesDaoImpl.java | 9 +++++++- .../java/org/sunbird/learner/util/Util.java | 23 +++++++++++++++++++ .../enrolments/CourseEnrolmentActor.scala | 3 ++- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/CourseBatchDaoImpl.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/CourseBatchDaoImpl.java index 0ad551cc..8cfc0844 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/CourseBatchDaoImpl.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/CourseBatchDaoImpl.java @@ -31,7 +31,7 @@ public class CourseBatchDaoImpl implements CourseBatchDao { @Override public Response create(RequestContext requestContext, CourseBatch courseBatch) { Map map = CourseBatchUtil.cassandraCourseMapping(courseBatch, dateFormat); - map = CassandraUtil.changeCassandraColumnMapping(map); + map = Util.toCollectionColumns(CassandraUtil.changeCassandraColumnMapping(map)); return cassandraOperation.insertRecord( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), map, requestContext); } @@ -41,11 +41,12 @@ public Response update(RequestContext requestContext, String courseId, String ba Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); + Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Map attributeMap = new HashMap<>(); attributeMap.putAll(map); attributeMap.remove(JsonKey.COURSE_ID); attributeMap.remove(JsonKey.BATCH_ID); - attributeMap = CassandraUtil.changeCassandraColumnMapping(attributeMap); + attributeMap = Util.toCollectionColumns(CassandraUtil.changeCassandraColumnMapping(attributeMap)); return cassandraOperation.updateRecord( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), attributeMap, primaryKey, requestContext); } @@ -55,6 +56,7 @@ public CourseBatch readById(String courseId, String batchId, RequestContext requ Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); + Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Response courseBatchResult = cassandraOperation.getRecordByIdentifier( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), primaryKey, null, requestContext); @@ -76,6 +78,7 @@ public Map getCourseBatch(RequestContext requestContext, String Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); + Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Response courseBatchResult = cassandraOperation.getRecordByIdentifier( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), primaryKey, null, requestContext); @@ -96,6 +99,7 @@ public void addCertificateTemplateToCourseBatch( Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); + Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) cassandraOperation.updateAddMapRecord( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), @@ -111,6 +115,7 @@ public void removeCertificateTemplateFromCourseBatch( Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); + Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) cassandraOperation.updateRemoveMapRecord( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/UserCoursesDaoImpl.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/UserCoursesDaoImpl.java index f65bbf01..eb55dedf 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/UserCoursesDaoImpl.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/UserCoursesDaoImpl.java @@ -35,6 +35,7 @@ public UserCourses read(RequestContext requestContext, String batchId, String us Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.BATCH_ID, batchId); primaryKey.put(JsonKey.USER_ID, userId); + Util.toCollectionColumns(primaryKey); // viewer: batchid->contextid (no-op when disabled) Response response = cassandraOperation.getRecordByIdentifier(KEYSPACE_NAME, TABLE_NAME, primaryKey, null, requestContext); List> userCoursesList = (List>) response.get(JsonKey.RESPONSE); @@ -54,6 +55,7 @@ public Response update(RequestContext requestContext, String batchId, String use Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.BATCH_ID, batchId); primaryKey.put(JsonKey.USER_ID, userId); + Util.toCollectionColumns(primaryKey); // viewer: batchid->contextid (no-op when disabled) Map updateList = new HashMap<>(); updateList.putAll(updateAttributes); updateList.remove(JsonKey.BATCH_ID); @@ -87,6 +89,7 @@ public Response updateV2(RequestContext requestContext, String userId, String co primaryKey.put(JsonKey.USER_ID, userId); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); + Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Map updateList = new HashMap<>(); updateList.putAll(updateAttributes); updateList.remove(JsonKey.BATCH_ID_KEY); @@ -101,6 +104,7 @@ public UserCourses read(RequestContext requestContext, String userId, String cou primaryKey.put(JsonKey.USER_ID, userId); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); + Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Response response = cassandraOperation.getRecordByIdentifier(KEYSPACE_NAME, USER_ENROLMENTS, primaryKey, null, requestContext); List> userCoursesList = (List>) response.get(JsonKey.RESPONSE); @@ -119,7 +123,9 @@ public List getBatchParticipants(RequestContext requestContext, String b Map queryMap = new HashMap<>(); queryMap.put(JsonKey.BATCH_ID, batchId); Response response = - cassandraOperation.getRecordsByIndexedProperty(KEYSPACE_NAME, USER_ENROLMENTS, "batchid", batchId, requestContext); + // viewer: user_enrolments is indexed on collectionid (batchid->contextid has NO index in the + // generalised schema) — this by-batch query needs a contextid index added to work under viewer. + cassandraOperation.getRecordsByIndexedProperty(KEYSPACE_NAME, USER_ENROLMENTS, Util.VIEWER_ENABLED ? "contextid" : "batchid", batchId, requestContext); /*cassandraOperation.getRecords( requestContext, KEYSPACE_NAME, USER_ENROLMENTS, queryMap, Arrays.asList(JsonKey.USER_ID, JsonKey.ACTIVE));*/ List> userCoursesList = @@ -141,6 +147,7 @@ public List> listEnrolments(RequestContext requestContext, S if(!CollectionUtils.isEmpty(courseIdList)){ primaryKey.put(JsonKey.COURSE_ID_KEY, courseIdList); } + Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid (no-op when disabled) Response response = cassandraOperation.getRecordByIdentifier(KEYSPACE_NAME, USER_ENROLMENTS, primaryKey, null, requestContext); List> userCoursesList = (List>) response.get(JsonKey.RESPONSE); if (CollectionUtils.isEmpty(userCoursesList)) { diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/Util.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/Util.java index b9afd96f..6b0441d4 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/Util.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/Util.java @@ -99,6 +99,29 @@ private static Map manualMapCopy(Object mapObject) { initializeDBProperty(); } + /** + * When viewer_enabled the shared course tables are GENERALISED to a collection identity — + * courseid -> collectionid, batchid -> contextid on course_batch / user_enrolments / + * user_content_consumption. The legacy course DAOs still address these tables by courseid/batchid, + * so this remaps the identifier keys (camelCase JsonKey or physical column form) to the generalised + * column names in a query/attribute map. No-op when viewer is disabled (legacy schema). + */ + public static final boolean VIEWER_ENABLED = + Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")); + + public static Map toCollectionColumns(Map map) { + if (!VIEWER_ENABLED || map == null) return map; + moveKey(map, JsonKey.COURSE_ID, "collectionid"); // "courseId" + moveKey(map, JsonKey.COURSE_ID_KEY, "collectionid"); // "courseid" + moveKey(map, JsonKey.BATCH_ID, "contextid"); // "batchId" + moveKey(map, JsonKey.BATCH_ID_KEY, "contextid"); // "batchid" + return map; + } + + private static void moveKey(Map map, String from, String to) { + if (map.containsKey(from)) map.put(to, map.remove(from)); + } + private Util() {} /** This method will initialize the cassandra data base property */ diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index dcfd5f12..9769380d 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -265,7 +265,8 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def upsertEnrollment(userId: String, courseId: String, batchId: String, data: java.util.Map[String, AnyRef], isNew: Boolean, requestContext: RequestContext): Unit = { val dataMap = CassandraUtil.changeCassandraColumnMapping(data) if(isNew) { - userCoursesDao.insertV2(requestContext, dataMap) + // viewer: remap courseid->collectionid, batchid->contextid on the insert row (no-op when disabled). + userCoursesDao.insertV2(requestContext, Util.toCollectionColumns(dataMap)) } else { userCoursesDao.updateV2(requestContext, userId, courseId, batchId, dataMap) } From 674a7b7eea566a899a93f4bb8cc3420489b94d9d Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 4 Aug 2026 10:49:45 +0530 Subject: [PATCH 08/30] feat(coursebatch): use collectionId/contextId for the courseBatch ES index Point the courseBatch ES read/write at the generalised field names (course-batch-1): - CourseBatchUtil.esCourseMapping writes collectionId/contextId (batch create/update indexing) - CourseBatchUtil.validateCourseBatch reads collectionId - BaseEnrolmentActor.getBatches filters by contextId - CourseEnrolmentActor.addBatchDetails keys batch docs by contextId - CourseBatchController.search translates client courseId/batchId filters to collectionId/contextId --- .../main/java/org/sunbird/learner/util/CourseBatchUtil.java | 5 ++++- .../scala/org/sunbird/enrolments/BaseEnrolmentActor.scala | 2 +- .../scala/org/sunbird/enrolments/CourseEnrolmentActor.scala | 2 +- .../controllers/coursemanagement/CourseBatchController.java | 6 +++++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java index 7120f86e..ffe90300 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java @@ -54,7 +54,7 @@ public static Map validateCourseBatch(RequestContext requestCont ProjectCommonException.throwClientErrorException(ResponseCode.CLIENT_ERROR, "No such batchId exists"); } if (StringUtils.isNotBlank(courseId) - && !StringUtils.equals(courseId, (String) result.get(JsonKey.COURSE_ID))) { + && !StringUtils.equals(courseId, (String) result.get("collectionId"))) { ProjectCommonException.throwClientErrorException(ResponseCode.CLIENT_ERROR, "batchId is not linked with courseId"); } return result; @@ -212,6 +212,9 @@ public static Map esCourseMapping(CourseBatch courseBatch, Strin }); esCourseMap.put(CourseJsonKey.CERTIFICATE_TEMPLATES_COLUMN, courseBatch.getCertTemplates()); + // courseBatch ES index uses generalised identity fields: courseId -> collectionId, batchId -> contextId + if (esCourseMap.containsKey(JsonKey.COURSE_ID)) esCourseMap.put("collectionId", esCourseMap.remove(JsonKey.COURSE_ID)); + if (esCourseMap.containsKey(JsonKey.BATCH_ID)) esCourseMap.put("contextId", esCourseMap.remove(JsonKey.BATCH_ID)); return esCourseMap; } diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala index 8f7c3964..91d89abb 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala @@ -19,7 +19,7 @@ abstract class BaseEnrolmentActor extends BaseActor { def getBatches(requestContext: RequestContext, batchIds: java.util.List[String], requestedFields: java.util.List[String]): java.util.List[java.util.Map[String, AnyRef]] = { val dto = new SearchDTO dto.setLimit(batchIds.size()) - dto.getAdditionalProperties().put(JsonKey.FILTERS, new java.util.HashMap[String, AnyRef](){{ put(JsonKey.BATCH_ID, batchIds)}}) + dto.getAdditionalProperties().put(JsonKey.FILTERS, new java.util.HashMap[String, AnyRef](){{ put("contextId", batchIds)}}) if(CollectionUtils.isNotEmpty(requestedFields)) dto.setFields(requestedFields) val future = esService.search(dto, ProjectUtil.EsType.courseBatch.getTypeName, requestContext) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index 9769380d..f3041e33 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -215,7 +215,7 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c val batchDetails = searchBatchDetails(batchIds, request) if(CollectionUtils.isNotEmpty(batchDetails)){ batchDetails.foreach(batch => CourseBatchUtil.enrichBatchStatusFromDates(batch)) - val batchMap = batchDetails.map(b => b.get(JsonKey.BATCH_ID).asInstanceOf[String] -> b).toMap + val batchMap = batchDetails.map(b => b.get("contextId").asInstanceOf[String] -> b).toMap enrolmentList.map(enrolment => { enrolment.put(JsonKey.BATCH, batchMap.getOrElse(enrolment.get(JsonKey.BATCH_ID).asInstanceOf[String], new java.util.HashMap[String, AnyRef]())) //To Do : A temporary change to support updation of completed course remove in next release diff --git a/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java b/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java index cc025784..3a39d577 100644 --- a/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java +++ b/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java @@ -98,7 +98,11 @@ public CompletionStage search(Http.Request httpRequest) { if (reqObj.getRequest().containsKey(JsonKey.FILTERS) && reqObj.getRequest().get(JsonKey.FILTERS) != null && reqObj.getRequest().get(JsonKey.FILTERS) instanceof Map) { - ((Map) (reqObj.getRequest().get(JsonKey.FILTERS))).put(JsonKey.OBJECT_TYPE, esObjectType); + Map f = (Map) reqObj.getRequest().get(JsonKey.FILTERS); + // courseBatch ES index uses generalised identity fields; translate client filter keys + if (f.containsKey(JsonKey.COURSE_ID)) f.put("collectionId", f.remove(JsonKey.COURSE_ID)); + if (f.containsKey(JsonKey.BATCH_ID)) f.put("contextId", f.remove(JsonKey.BATCH_ID)); + f.put(JsonKey.OBJECT_TYPE, esObjectType); } else { Map filtermap = new HashMap<>(); Map dataMap = new HashMap<>(); From 4e06a65bcf0584f2f9c43205ee37a8243afa50bd Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 4 Aug 2026 11:03:26 +0530 Subject: [PATCH 09/30] feat(coursebatch): gate collectionId/contextId ES fields on viewer_enabled Non-viewer deployments keep courseId/batchId (old course-batch index); viewer deployments use collectionId/contextId (course-batch-1). Gated via Util.VIEWER_ENABLED in: - CourseBatchUtil.esCourseMapping (write) + validateCourseBatch (read) - BaseEnrolmentActor.getBatches (search filter) + CourseEnrolmentActor.addBatchDetails (doc key) - CourseBatchController.search (client filter translation) Adds Util import to BaseEnrolmentActor.scala and CourseBatchController.java. --- .../java/org/sunbird/learner/util/CourseBatchUtil.java | 10 ++++++---- .../org/sunbird/enrolments/BaseEnrolmentActor.scala | 4 +++- .../org/sunbird/enrolments/CourseEnrolmentActor.scala | 3 ++- .../coursemanagement/CourseBatchController.java | 9 ++++++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java index ffe90300..60cdba98 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java @@ -54,7 +54,7 @@ public static Map validateCourseBatch(RequestContext requestCont ProjectCommonException.throwClientErrorException(ResponseCode.CLIENT_ERROR, "No such batchId exists"); } if (StringUtils.isNotBlank(courseId) - && !StringUtils.equals(courseId, (String) result.get("collectionId"))) { + && !StringUtils.equals(courseId, (String) result.get(Util.VIEWER_ENABLED ? "collectionId" : JsonKey.COURSE_ID))) { ProjectCommonException.throwClientErrorException(ResponseCode.CLIENT_ERROR, "batchId is not linked with courseId"); } return result; @@ -212,9 +212,11 @@ public static Map esCourseMapping(CourseBatch courseBatch, Strin }); esCourseMap.put(CourseJsonKey.CERTIFICATE_TEMPLATES_COLUMN, courseBatch.getCertTemplates()); - // courseBatch ES index uses generalised identity fields: courseId -> collectionId, batchId -> contextId - if (esCourseMap.containsKey(JsonKey.COURSE_ID)) esCourseMap.put("collectionId", esCourseMap.remove(JsonKey.COURSE_ID)); - if (esCourseMap.containsKey(JsonKey.BATCH_ID)) esCourseMap.put("contextId", esCourseMap.remove(JsonKey.BATCH_ID)); + // viewer.enabled: courseBatch ES index uses generalised identity fields (courseId->collectionId, batchId->contextId) + if (Util.VIEWER_ENABLED) { + if (esCourseMap.containsKey(JsonKey.COURSE_ID)) esCourseMap.put("collectionId", esCourseMap.remove(JsonKey.COURSE_ID)); + if (esCourseMap.containsKey(JsonKey.BATCH_ID)) esCourseMap.put("contextId", esCourseMap.remove(JsonKey.BATCH_ID)); + } return esCourseMap; } diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala index 91d89abb..f68f5a74 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala @@ -7,6 +7,7 @@ import org.sunbird.common.factory.EsClientFactory import org.sunbird.common.inf.ElasticSearchService import org.sunbird.keys.JsonKey import org.sunbird.common.ProjectUtil +import org.sunbird.learner.util.Util import org.sunbird.request.RequestContext import org.sunbird.dto.SearchDTO @@ -19,7 +20,8 @@ abstract class BaseEnrolmentActor extends BaseActor { def getBatches(requestContext: RequestContext, batchIds: java.util.List[String], requestedFields: java.util.List[String]): java.util.List[java.util.Map[String, AnyRef]] = { val dto = new SearchDTO dto.setLimit(batchIds.size()) - dto.getAdditionalProperties().put(JsonKey.FILTERS, new java.util.HashMap[String, AnyRef](){{ put("contextId", batchIds)}}) + val batchField = if (Util.VIEWER_ENABLED) "contextId" else JsonKey.BATCH_ID + dto.getAdditionalProperties().put(JsonKey.FILTERS, new java.util.HashMap[String, AnyRef](){{ put(batchField, batchIds)}}) if(CollectionUtils.isNotEmpty(requestedFields)) dto.setFields(requestedFields) val future = esService.search(dto, ProjectUtil.EsType.courseBatch.getTypeName, requestContext) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index f3041e33..dd1e98d2 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -215,7 +215,8 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c val batchDetails = searchBatchDetails(batchIds, request) if(CollectionUtils.isNotEmpty(batchDetails)){ batchDetails.foreach(batch => CourseBatchUtil.enrichBatchStatusFromDates(batch)) - val batchMap = batchDetails.map(b => b.get("contextId").asInstanceOf[String] -> b).toMap + val batchField = if (Util.VIEWER_ENABLED) "contextId" else JsonKey.BATCH_ID + val batchMap = batchDetails.map(b => b.get(batchField).asInstanceOf[String] -> b).toMap enrolmentList.map(enrolment => { enrolment.put(JsonKey.BATCH, batchMap.getOrElse(enrolment.get(JsonKey.BATCH_ID).asInstanceOf[String], new java.util.HashMap[String, AnyRef]())) //To Do : A temporary change to support updation of completed course remove in next release diff --git a/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java b/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java index 3a39d577..279cfff2 100644 --- a/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java +++ b/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java @@ -7,6 +7,7 @@ import controllers.coursemanagement.validator.CourseBatchRequestValidator; import org.sunbird.operations.lms.ActorOperations; import org.sunbird.keys.JsonKey; +import org.sunbird.learner.util.Util; import org.sunbird.common.ProjectUtil.EsType; import org.sunbird.request.Request; import play.mvc.Http; @@ -99,9 +100,11 @@ public CompletionStage search(Http.Request httpRequest) { && reqObj.getRequest().get(JsonKey.FILTERS) != null && reqObj.getRequest().get(JsonKey.FILTERS) instanceof Map) { Map f = (Map) reqObj.getRequest().get(JsonKey.FILTERS); - // courseBatch ES index uses generalised identity fields; translate client filter keys - if (f.containsKey(JsonKey.COURSE_ID)) f.put("collectionId", f.remove(JsonKey.COURSE_ID)); - if (f.containsKey(JsonKey.BATCH_ID)) f.put("contextId", f.remove(JsonKey.BATCH_ID)); + // viewer.enabled: courseBatch ES index uses generalised identity fields; translate client filter keys + if (Util.VIEWER_ENABLED) { + if (f.containsKey(JsonKey.COURSE_ID)) f.put("collectionId", f.remove(JsonKey.COURSE_ID)); + if (f.containsKey(JsonKey.BATCH_ID)) f.put("contextId", f.remove(JsonKey.BATCH_ID)); + } f.put(JsonKey.OBJECT_TYPE, esObjectType); } else { Map filtermap = new HashMap<>(); From 4af39c6a21d96d7393587eaad5973d5ee5a2bbe4 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 4 Aug 2026 11:35:16 +0530 Subject: [PATCH 10/30] feat(enrolment): Map viewer docs to courseId/batchId When Util.VIEWER_ENABLED, translate stored viewer fields (collectionId/contextId) back to the API contract (courseId/batchId). Changes added in SearchHandlerActor and CourseEnrolmentActor to remap keys so clients and downstream logic (participants, status) remain unchanged. Also adjust batchMap construction to use JsonKey.BATCH_ID. Status enrichment from dates is preserved. --- .../learner/actors/search/SearchHandlerActor.java | 9 +++++++++ .../org/sunbird/enrolments/CourseEnrolmentActor.scala | 8 ++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/search/SearchHandlerActor.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/search/SearchHandlerActor.java index bb7990e8..4cba52ad 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/search/SearchHandlerActor.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/search/SearchHandlerActor.java @@ -120,6 +120,15 @@ public void onReceive(Request request) throws Throwable { if (EsType.courseBatch.getTypeName().equalsIgnoreCase(filterObjectType)) { List> courseBatchList = (List>) result.get(JsonKey.CONTENT); + // viewer.enabled: courseBatch docs are stored with collectionId/contextId; map back to the API + // contract (courseId/batchId) so clients and downstream (participants, status) are unchanged. + if (Util.VIEWER_ENABLED && CollectionUtils.isNotEmpty(courseBatchList)) { + courseBatchList.forEach(b -> { + if (b.containsKey("collectionId")) b.put(JsonKey.COURSE_ID, b.remove("collectionId")); + if (b.containsKey("contextId")) b.put(JsonKey.BATCH_ID, b.remove("contextId")); + }); + } + // Recompute status from dates for all search results to handle stale cached values if (CollectionUtils.isNotEmpty(courseBatchList)) { courseBatchList.forEach(batch -> CourseBatchUtil.enrichBatchStatusFromDates(batch)); diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index dd1e98d2..2db8c180 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -215,8 +215,12 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c val batchDetails = searchBatchDetails(batchIds, request) if(CollectionUtils.isNotEmpty(batchDetails)){ batchDetails.foreach(batch => CourseBatchUtil.enrichBatchStatusFromDates(batch)) - val batchField = if (Util.VIEWER_ENABLED) "contextId" else JsonKey.BATCH_ID - val batchMap = batchDetails.map(b => b.get(batchField).asInstanceOf[String] -> b).toMap + // viewer.enabled: batch docs are stored with collectionId/contextId; map back to the API contract courseId/batchId + if (Util.VIEWER_ENABLED) batchDetails.foreach { b => + if (b.containsKey("collectionId")) b.put(JsonKey.COURSE_ID, b.remove("collectionId")) + if (b.containsKey("contextId")) b.put(JsonKey.BATCH_ID, b.remove("contextId")) + } + val batchMap = batchDetails.map(b => b.get(JsonKey.BATCH_ID).asInstanceOf[String] -> b).toMap enrolmentList.map(enrolment => { enrolment.put(JsonKey.BATCH, batchMap.getOrElse(enrolment.get(JsonKey.BATCH_ID).asInstanceOf[String], new java.util.HashMap[String, AnyRef]())) //To Do : A temporary change to support updation of completed course remove in next release From 727121a7bbf547108861bd22d49b217639730a8f Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 4 Aug 2026 14:43:48 +0530 Subject: [PATCH 11/30] fix: map renamed collectionid/contextid columns to collectionId/contextId Adds collectionid=collectionId and contextid=contextId to the Cassandra column->field mapping so viewer-renamed columns resolve to the camelCase field names used by ES sync, batch search, and the enrolment list. Inert on non-viewer clusters (columns absent). --- .../src/main/resources/cassandratablecolumn.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/sunbird-cassandra-utils/src/main/resources/cassandratablecolumn.properties b/core/sunbird-cassandra-utils/src/main/resources/cassandratablecolumn.properties index 64a76611..ebceac6e 100644 --- a/core/sunbird-cassandra-utils/src/main/resources/cassandratablecolumn.properties +++ b/core/sunbird-cassandra-utils/src/main/resources/cassandratablecolumn.properties @@ -51,6 +51,8 @@ courseadditionalinfo=courseAdditionalInfo coursecreator=courseCreator courseduration=courseDuration courseid=courseId +collectionid=collectionId +contextid=contextId courselogourl=courseLogoUrl coursename=courseName courseversion=courseVersion From 59f032d5c19b11f55780f89e48e046c136161c48 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 4 Aug 2026 15:50:26 +0530 Subject: [PATCH 12/30] fix: restore courseId/batchId on enrolment rows for /user/courses/list Viewer-renamed columns read back as collectionId/contextId; remap them to the API contract courseId/batchId at the enrolment read point so the list flow (course/batch details join) resolves. Gated on viewer_enabled. --- .../scala/org/sunbird/enrolments/CourseEnrolmentActor.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index 2db8c180..7af964eb 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -160,6 +160,11 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def getActiveEnrollments(userId: String, courseIdList: java.util.List[String], requestContext: RequestContext): java.util.List[java.util.Map[String, AnyRef]] = { val enrolments: java.util.List[java.util.Map[String, AnyRef]] = userCoursesDao.listEnrolments(requestContext, userId, courseIdList) + // viewer.enabled: enrolment rows read back as collectionId/contextId; restore the API contract courseId/batchId + if (Util.VIEWER_ENABLED) enrolments.forEach(e => { + if (e.containsKey("collectionId")) e.put(JsonKey.COURSE_ID, e.remove("collectionId")) + if (e.containsKey("contextId")) e.put(JsonKey.BATCH_ID, e.remove("contextId")) + }) if (CollectionUtils.isNotEmpty(enrolments)) { val activeEnrolments = enrolments.filter(e => e.getOrDefault(JsonKey.ACTIVE, false.asInstanceOf[AnyRef]).asInstanceOf[Boolean]) val sortedEnrolment = activeEnrolments.filter(ae => ae.get(JsonKey.COURSE_ENROLL_DATE)!=null).toList.sortBy(_.get(JsonKey.COURSE_ENROLL_DATE).asInstanceOf[Date])(Ordering[Date].reverse).toList From 2d4acca595f66142b8d0b5070577d3545e99400f Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 4 Aug 2026 16:12:45 +0530 Subject: [PATCH 13/30] fix: guard enrolment-row remap behind null/empty check listEnrolments returns null (not empty) when a user has no enrolments; move the viewer collectionId/contextId->courseId/batchId remap inside the existing isNotEmpty guard to avoid NPE. --- .../org/sunbird/enrolments/CourseEnrolmentActor.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index 7af964eb..998ead59 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -160,12 +160,12 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def getActiveEnrollments(userId: String, courseIdList: java.util.List[String], requestContext: RequestContext): java.util.List[java.util.Map[String, AnyRef]] = { val enrolments: java.util.List[java.util.Map[String, AnyRef]] = userCoursesDao.listEnrolments(requestContext, userId, courseIdList) - // viewer.enabled: enrolment rows read back as collectionId/contextId; restore the API contract courseId/batchId - if (Util.VIEWER_ENABLED) enrolments.forEach(e => { - if (e.containsKey("collectionId")) e.put(JsonKey.COURSE_ID, e.remove("collectionId")) - if (e.containsKey("contextId")) e.put(JsonKey.BATCH_ID, e.remove("contextId")) - }) if (CollectionUtils.isNotEmpty(enrolments)) { + // viewer.enabled: enrolment rows read back as collectionId/contextId; restore the API contract courseId/batchId + if (Util.VIEWER_ENABLED) enrolments.forEach(e => { + if (e.containsKey("collectionId")) e.put(JsonKey.COURSE_ID, e.remove("collectionId")) + if (e.containsKey("contextId")) e.put(JsonKey.BATCH_ID, e.remove("contextId")) + }) val activeEnrolments = enrolments.filter(e => e.getOrDefault(JsonKey.ACTIVE, false.asInstanceOf[AnyRef]).asInstanceOf[Boolean]) val sortedEnrolment = activeEnrolments.filter(ae => ae.get(JsonKey.COURSE_ENROLL_DATE)!=null).toList.sortBy(_.get(JsonKey.COURSE_ENROLL_DATE).asInstanceOf[Date])(Ordering[Date].reverse).toList val finalEnrolments = sortedEnrolment ++ activeEnrolments.filter(e => e.get(JsonKey.COURSE_ENROLL_DATE)==null).toList From a27b9ed66cb65a51ebb4177a5f6b0451689a60bf Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 4 Aug 2026 17:28:47 +0530 Subject: [PATCH 14/30] chore: add trace logging across enrolment list flow Logs cache decision + hit/miss, raw read count and row keys, post-remap courseId/batchId, active-filter count, content-search request/result, course-filter kept count, batch join, and final returned count -- to pinpoint where /user/courses/list drops enrolments. --- .../enrolments/CourseEnrolmentActor.scala | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index 998ead59..f3acc708 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -145,9 +145,10 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def list(request: Request): Unit = { val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] val courseIdList = request.get(JsonKey.COURSE_IDS).asInstanceOf[java.util.List[String]] - logger.info(request.getRequestContext,"CourseEnrolmentActor :: list :: UserId = " + userId) + val useCache = isCacheEnabled && request.getContext.get("cache").asInstanceOf[Boolean] + logger.info(request.getRequestContext,"CourseEnrolmentActor :: list :: UserId = " + userId + " courseIdList=" + courseIdList + " isCacheEnabled=" + isCacheEnabled + " contextCacheFlag=" + request.getContext.get("cache") + " => useCache=" + useCache) try{ - val response = if (isCacheEnabled && request.getContext.get("cache").asInstanceOf[Boolean]) + val response = if (useCache) getCachedEnrolmentList(userId, () => getEnrolmentList(request, userId, courseIdList)) else getEnrolmentList(request, userId, courseIdList) sender().tell(response, self) }catch { @@ -160,32 +161,40 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def getActiveEnrollments(userId: String, courseIdList: java.util.List[String], requestContext: RequestContext): java.util.List[java.util.Map[String, AnyRef]] = { val enrolments: java.util.List[java.util.Map[String, AnyRef]] = userCoursesDao.listEnrolments(requestContext, userId, courseIdList) + logger.info(requestContext, "getActiveEnrollments :: userId=" + userId + " viewerEnabled=" + Util.VIEWER_ENABLED + " rawEnrolments=" + (if (enrolments == null) "null" else enrolments.size.toString) + " firstRowKeys=" + (if (CollectionUtils.isNotEmpty(enrolments)) enrolments.get(0).keySet.toString else "[]")) if (CollectionUtils.isNotEmpty(enrolments)) { // viewer.enabled: enrolment rows read back as collectionId/contextId; restore the API contract courseId/batchId if (Util.VIEWER_ENABLED) enrolments.forEach(e => { if (e.containsKey("collectionId")) e.put(JsonKey.COURSE_ID, e.remove("collectionId")) if (e.containsKey("contextId")) e.put(JsonKey.BATCH_ID, e.remove("contextId")) }) + logger.info(requestContext, "getActiveEnrollments :: after viewer remap :: firstRow courseId=" + enrolments.get(0).get(JsonKey.COURSE_ID) + " batchId=" + enrolments.get(0).get(JsonKey.BATCH_ID) + " active=" + enrolments.get(0).get(JsonKey.ACTIVE)) val activeEnrolments = enrolments.filter(e => e.getOrDefault(JsonKey.ACTIVE, false.asInstanceOf[AnyRef]).asInstanceOf[Boolean]) + logger.info(requestContext, "getActiveEnrollments :: activeEnrolments(after active=true filter)=" + activeEnrolments.size) val sortedEnrolment = activeEnrolments.filter(ae => ae.get(JsonKey.COURSE_ENROLL_DATE)!=null).toList.sortBy(_.get(JsonKey.COURSE_ENROLL_DATE).asInstanceOf[Date])(Ordering[Date].reverse).toList val finalEnrolments = sortedEnrolment ++ activeEnrolments.filter(e => e.get(JsonKey.COURSE_ENROLL_DATE)==null).toList + logger.info(requestContext, "getActiveEnrollments :: finalEnrolments=" + finalEnrolments.size + " (limit=" + ProjectUtil.getConfigValue("enrollment_list_size") + ")") finalEnrolments.take(Integer.parseInt(ProjectUtil.getConfigValue("enrollment_list_size"))).toList.asJava } else { + logger.info(requestContext, "getActiveEnrollments :: no enrolments read from cassandra for userId=" + userId) new util.ArrayList[java.util.Map[String, AnyRef]]() } } def addCourseDetails(activeEnrolments: java.util.List[java.util.Map[String, AnyRef]], courseIds: java.util.List[String] , request:Request): java.util.List[java.util.Map[String, AnyRef]] = { val requestBody: String = prepareSearchRequest(courseIds, request) + logger.info(request.getRequestContext, "addCourseDetails :: courseIds=" + courseIds + " searchRequestBody=" + requestBody) val searchResult:java.util.Map[String, AnyRef] = ContentSearchUtil.searchContentSync(request.getRequestContext, request.getContext.getOrDefault(JsonKey.URL_QUERY_STRING,"").asInstanceOf[String], requestBody, request.get(JsonKey.HEADER).asInstanceOf[java.util.Map[String, String]]) + logger.info(request.getRequestContext, "addCourseDetails :: searchResult=" + (if (searchResult == null) "NULL(search call failed)" else "keys=" + searchResult.keySet)) val coursesList: java.util.List[java.util.Map[String, AnyRef]] = searchResult.getOrDefault(JsonKey.CONTENTS, new java.util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] val coursesMap = { if(CollectionUtils.isNotEmpty(coursesList)) { coursesList.map(ev => ev.get(JsonKey.IDENTIFIER).asInstanceOf[String] -> ev).toMap } else Map() } - activeEnrolments.filter(enrolment => coursesMap.containsKey(enrolment.get(JsonKey.COURSE_ID))).map(enrolment => { + logger.info(request.getRequestContext, "addCourseDetails :: coursesList=" + (if (coursesList == null) "null" else coursesList.size.toString) + " coursesMapKeys=" + coursesMap.keySet + " enrolmentCourseIds=" + activeEnrolments.map(e => e.get(JsonKey.COURSE_ID)).mkString("[", ",", "]")) + val withCourse = activeEnrolments.filter(enrolment => coursesMap.containsKey(enrolment.get(JsonKey.COURSE_ID))).map(enrolment => { val courseContent = coursesMap.get(enrolment.get(JsonKey.COURSE_ID)) enrolment.put(JsonKey.COURSE_NAME, courseContent.get(JsonKey.NAME)) enrolment.put(JsonKey.DESCRIPTION, courseContent.get(JsonKey.DESCRIPTION)) @@ -196,6 +205,8 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c enrolment.put(JsonKey.CONTENT, courseContent) enrolment }).toList.asJava + logger.info(request.getRequestContext, "addCourseDetails :: keptAfterCourseFilter=" + withCourse.size) + withCourse } def prepareSearchRequest(courseIds: java.util.List[String], request: Request): String = { @@ -218,6 +229,7 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def addBatchDetails(enrolmentList: util.List[util.Map[String, AnyRef]], request: Request): util.List[util.Map[String, AnyRef]] = { val batchIds:java.util.List[String] = enrolmentList.map(e => e.getOrDefault(JsonKey.BATCH_ID, "").asInstanceOf[String]).distinct.filter(id => StringUtils.isNotBlank(id)).toList.asJava val batchDetails = searchBatchDetails(batchIds, request) + logger.info(request.getRequestContext, "addBatchDetails :: enrolmentListSize=" + enrolmentList.size + " batchIds=" + batchIds + " batchDetailsFound=" + (if (batchDetails == null) "null" else batchDetails.size.toString)) if(CollectionUtils.isNotEmpty(batchDetails)){ batchDetails.foreach(batch => CourseBatchUtil.enrichBatchStatusFromDates(batch)) // viewer.enabled: batch docs are stored with collectionId/contextId; map back to the API contract courseId/batchId @@ -353,8 +365,10 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c val key = getCacheKey(userId) val responseString = cacheUtil.get(key) if (StringUtils.isNotBlank(responseString)) { + logger.info(null.asInstanceOf[RequestContext], "getCachedEnrolmentList :: CACHE HIT key=" + key + " (serving cached response, len=" + responseString.length + ")") JsonUtil.deserialize(responseString, classOf[Response]) } else { + logger.info(null.asInstanceOf[RequestContext], "getCachedEnrolmentList :: CACHE MISS key=" + key + " -> querying cassandra") val response = handleEmptyCache() val responseString = JsonUtil.serialize(response) cacheUtil.set(key, responseString, ttl) @@ -379,6 +393,7 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c val resp: Response = new Response() val sortedEnrolment = enrolments.filter(ae => ae.get("lastContentAccessTime")!=null).toList.sortBy(_.get("lastContentAccessTime").asInstanceOf[Date])(Ordering[Date].reverse).toList val finalEnrolments = sortedEnrolment ++ enrolments.asScala.filter(e => e.get("lastContentAccessTime")==null).toList + logger.info(request.getRequestContext, "getEnrolmentList :: userId=" + userId + " FINAL courses returned=" + finalEnrolments.size) resp.put(JsonKey.COURSES, finalEnrolments.asJava) resp } From 2c2b3469df1305f16f73497835af2c536db46247 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 4 Aug 2026 17:54:38 +0530 Subject: [PATCH 15/30] fix: remap lowercase collectionid/contextid on enrolment rows too Logs confirmed the read returns lowercase collectionid/contextid when the column-mapping properties aren't deployed, so the camelCase-only remap left courseId/batchId null and the list dropped every enrolment. Handle both camelCase and lowercase forms. --- .../scala/org/sunbird/enrolments/CourseEnrolmentActor.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index f3acc708..c91e1673 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -163,10 +163,13 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c val enrolments: java.util.List[java.util.Map[String, AnyRef]] = userCoursesDao.listEnrolments(requestContext, userId, courseIdList) logger.info(requestContext, "getActiveEnrollments :: userId=" + userId + " viewerEnabled=" + Util.VIEWER_ENABLED + " rawEnrolments=" + (if (enrolments == null) "null" else enrolments.size.toString) + " firstRowKeys=" + (if (CollectionUtils.isNotEmpty(enrolments)) enrolments.get(0).keySet.toString else "[]")) if (CollectionUtils.isNotEmpty(enrolments)) { - // viewer.enabled: enrolment rows read back as collectionId/contextId; restore the API contract courseId/batchId + // viewer.enabled: enrolment rows read back as collectionId/contextId (camelCase when the column-mapping + // properties are deployed, else lowercase collectionid/contextid); restore the API contract courseId/batchId. if (Util.VIEWER_ENABLED) enrolments.forEach(e => { if (e.containsKey("collectionId")) e.put(JsonKey.COURSE_ID, e.remove("collectionId")) + else if (e.containsKey("collectionid")) e.put(JsonKey.COURSE_ID, e.remove("collectionid")) if (e.containsKey("contextId")) e.put(JsonKey.BATCH_ID, e.remove("contextId")) + else if (e.containsKey("contextid")) e.put(JsonKey.BATCH_ID, e.remove("contextid")) }) logger.info(requestContext, "getActiveEnrollments :: after viewer remap :: firstRow courseId=" + enrolments.get(0).get(JsonKey.COURSE_ID) + " batchId=" + enrolments.get(0).get(JsonKey.BATCH_ID) + " active=" + enrolments.get(0).get(JsonKey.ACTIVE)) val activeEnrolments = enrolments.filter(e => e.getOrDefault(JsonKey.ACTIVE, false.asInstanceOf[AnyRef]).asInstanceOf[Boolean]) From f6c7d6bcfd8a785b139f49b54557915d3865384a Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 4 Aug 2026 19:18:53 +0530 Subject: [PATCH 16/30] fix: unify all courseBatch ES writers on collectionId/contextId Add idempotent CourseBatchUtil.toEsCollectionFields and call it in syncCourseBatchForeground (covers cert add/remove) and BackgroundJobManager.updateCourseBatchInfoToEs, so every courseBatch ES write emits camelCase collectionId/contextId under viewer. Prevents a secondary writer from overwriting the create doc with mismatched field names (the 'batch shows then disappears' bug). --- .../learner/actors/BackgroundJobManager.java | 2 ++ .../org/sunbird/learner/util/CourseBatchUtil.java | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/BackgroundJobManager.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/BackgroundJobManager.java index 4a49cff4..8593eac0 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/BackgroundJobManager.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/BackgroundJobManager.java @@ -16,6 +16,7 @@ import org.sunbird.response.ResponseCode; import org.sunbird.learner.actors.coursebatch.service.UserCoursesService; import org.sunbird.learner.util.CourseBatchSchedulerUtil; +import org.sunbird.learner.util.CourseBatchUtil; import org.sunbird.learner.util.Util; import scala.concurrent.Future; @@ -107,6 +108,7 @@ private void insertUserCourseInfoToEs(Request actorMessage) { @SuppressWarnings("unchecked") private void updateCourseBatchInfoToEs(Request actorMessage) { Map batch = (Map) actorMessage.getRequest().get(JsonKey.BATCH); + CourseBatchUtil.toEsCollectionFields(batch); // unify courseBatch ES writes on collectionId/contextId (viewer) updateDataToElastic(actorMessage.getRequestContext(), ProjectUtil.EsIndex.sunbird.getIndexName(), ProjectUtil.EsType.courseBatch.getTypeName(), (String) batch.get(JsonKey.ID), diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java index 60cdba98..5d72a67e 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java @@ -37,8 +37,21 @@ public class CourseBatchUtil { private CourseBatchUtil() {} + /** + * viewer.enabled: courseBatch ES docs use generalised identity fields (courseId->collectionId, + * batchId->contextId). Idempotent + no-op when viewer disabled, so it is safe to call on every + * courseBatch ES write path (create/update via esCourseMapping already renamed; cert/background + * writers pass a raw batch map and rely on this). + */ + public static void toEsCollectionFields(Map esMap) { + if (!Util.VIEWER_ENABLED || esMap == null) return; + if (esMap.containsKey(JsonKey.COURSE_ID)) esMap.put("collectionId", esMap.remove(JsonKey.COURSE_ID)); + if (esMap.containsKey(JsonKey.BATCH_ID)) esMap.put("contextId", esMap.remove(JsonKey.BATCH_ID)); + } + public static void syncCourseBatchForeground(RequestContext requestContext, String uniqueId, Map req) { logger.info(requestContext, "CourseBatchManagementActor: syncCourseBatchForeground called for course batch ID = " + uniqueId); + toEsCollectionFields(req); // unify all courseBatch ES writes on collectionId/contextId req.put(JsonKey.ID, uniqueId); req.put(JsonKey.IDENTIFIER, uniqueId); Future esResponseF = esUtil.save(ProjectUtil.EsType.courseBatch.getTypeName(), uniqueId, req, requestContext); From c5c55761cf5dfa82b55f8fcb9b1e7b99d94a334e Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Wed, 5 Aug 2026 11:55:00 +0530 Subject: [PATCH 17/30] refactor: use courseId/batchId everywhere; drop collectionId/contextId rename Revert the viewer collection-identity generalisation across shared lern code and the viewer module to the canonical courseId/batchId (courseid/ batchid columns, course_id/batch_id for assessment_aggregator). Removes Util.VIEWER_ENABLED/toCollectionColumns, CourseBatchUtil.toEsCollectionFields, the esCourseMapping/SearchHandlerActor/CourseBatchController remaps, the enrolment read remaps, and the assessment column gating. Viewer actors + controllers now speak courseId/batchId. user_activity_agg.context_id and the isViewerEnabled delegation toggle are unchanged. Adds a clean courseid/batchid test-keyspace CQL (viewer*.cql untouched). --- .../resources/cassandratablecolumn.properties | 2 - .../viewer/ViewAggregateController.java | 6 +- .../assessment/service/CassandraService.scala | 10 +- .../learner/actors/BackgroundJobManager.java | 2 - .../dao/impl/CourseBatchDaoImpl.java | 9 +- .../dao/impl/UserCoursesDaoImpl.java | 9 +- .../actors/search/SearchHandlerActor.java | 9 -- .../sunbird/learner/util/CourseBatchUtil.java | 20 +--- .../java/org/sunbird/learner/util/Util.java | 23 ----- .../enrolments/AssessmentAuditRecorder.scala | 7 +- .../enrolments/BaseEnrolmentActor.scala | 4 +- .../enrolments/ContentConsumptionActor.scala | 24 +++-- .../enrolments/CourseEnrolmentActor.scala | 37 +------ .../CourseBatchController.java | 10 -- .../viewer/actor/ViewConsumptionActor.scala | 54 +++++------ .../viewer/actor/ViewerAggregatorActor.scala | 96 +++++++++---------- .../viewer/actor/ViewerRequestKeys.scala | 6 +- .../viewer/actor/ViewerSummaryActor.scala | 30 +++--- .../viewer/ViewAggregateController.java | 6 +- .../service/app/util/RequestValidator.java | 4 +- 20 files changed, 125 insertions(+), 243 deletions(-) diff --git a/core/sunbird-cassandra-utils/src/main/resources/cassandratablecolumn.properties b/core/sunbird-cassandra-utils/src/main/resources/cassandratablecolumn.properties index ebceac6e..64a76611 100644 --- a/core/sunbird-cassandra-utils/src/main/resources/cassandratablecolumn.properties +++ b/core/sunbird-cassandra-utils/src/main/resources/cassandratablecolumn.properties @@ -51,8 +51,6 @@ courseadditionalinfo=courseAdditionalInfo coursecreator=courseCreator courseduration=courseDuration courseid=courseId -collectionid=collectionId -contextid=contextId courselogourl=courseLogoUrl coursename=courseName courseversion=courseVersion diff --git a/modules/lern/service/app/controllers/viewer/ViewAggregateController.java b/modules/lern/service/app/controllers/viewer/ViewAggregateController.java index a52762e7..362b8d5e 100644 --- a/modules/lern/service/app/controllers/viewer/ViewAggregateController.java +++ b/modules/lern/service/app/controllers/viewer/ViewAggregateController.java @@ -40,12 +40,12 @@ public CompletionStage agg(Http.Request httpRequest) { private void validate(Request request) { String userId = (String) request.get(JsonKey.USER_ID); - Object collectionId = request.get("collectionId") != null ? request.get("collectionId") : request.get(JsonKey.COURSE_ID); + Object courseId = request.get(JsonKey.COURSE_ID); if (userId == null || userId.trim().isEmpty() - || collectionId == null || collectionId.toString().trim().isEmpty()) { + || courseId == null || courseId.toString().trim().isEmpty()) { throw new ProjectCommonException( ResponseCode.mandatoryParamsMissing.getErrorCode(), - "userId and collectionId (or courseId) are mandatory", + "userId and courseId are mandatory", ResponseCode.CLIENT_ERROR.getResponseCode()); } } diff --git a/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala b/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala index 9b17d106..dd0880f4 100644 --- a/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala +++ b/modules/lms/assessment-aggregator/src/main/scala/org/sunbird/assessment/service/CassandraService.scala @@ -18,13 +18,9 @@ class CassandraService(optionalDao: Option[CassandraOperation] = None) { private val activityTable = Option(ProjectUtil.getConfigValue("user_activity_agg_table")).getOrElse("user_activity_agg") private lazy val questionType: UserType = dao.getUDTType(keyspace, Option(ProjectUtil.getConfigValue("assessment_question_udt_type")).getOrElse("question")) - // assessment_aggregator identity columns are gated by viewer_enabled: viewer ON -> generalised names - // (table migrated), viewer OFF -> legacy names (un-migrated table). Lets the legacy /v1/assessment/agg - // path keep working on the old schema when the viewer is disabled. (user_activity_agg.context_id below - // is a DIFFERENT column and is NOT gated.) - private def viewerEnabled: Boolean = java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) - private def collectionCol: String = if (viewerEnabled) "collection_id" else "course_id" - private def contextCol: String = if (viewerEnabled) "context_id" else "batch_id" + // assessment_aggregator identity columns. (user_activity_agg.context_id below is a DIFFERENT column.) + private val collectionCol: String = "course_id" + private val contextCol: String = "batch_id" def getAssessment(aid: String, uid: String, cid: String, bid: String, contId: String, ctx: RequestContext): Option[ExistingAssessment] = { try { diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/BackgroundJobManager.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/BackgroundJobManager.java index 8593eac0..4a49cff4 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/BackgroundJobManager.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/BackgroundJobManager.java @@ -16,7 +16,6 @@ import org.sunbird.response.ResponseCode; import org.sunbird.learner.actors.coursebatch.service.UserCoursesService; import org.sunbird.learner.util.CourseBatchSchedulerUtil; -import org.sunbird.learner.util.CourseBatchUtil; import org.sunbird.learner.util.Util; import scala.concurrent.Future; @@ -108,7 +107,6 @@ private void insertUserCourseInfoToEs(Request actorMessage) { @SuppressWarnings("unchecked") private void updateCourseBatchInfoToEs(Request actorMessage) { Map batch = (Map) actorMessage.getRequest().get(JsonKey.BATCH); - CourseBatchUtil.toEsCollectionFields(batch); // unify courseBatch ES writes on collectionId/contextId (viewer) updateDataToElastic(actorMessage.getRequestContext(), ProjectUtil.EsIndex.sunbird.getIndexName(), ProjectUtil.EsType.courseBatch.getTypeName(), (String) batch.get(JsonKey.ID), diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/CourseBatchDaoImpl.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/CourseBatchDaoImpl.java index 8cfc0844..0ad551cc 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/CourseBatchDaoImpl.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/CourseBatchDaoImpl.java @@ -31,7 +31,7 @@ public class CourseBatchDaoImpl implements CourseBatchDao { @Override public Response create(RequestContext requestContext, CourseBatch courseBatch) { Map map = CourseBatchUtil.cassandraCourseMapping(courseBatch, dateFormat); - map = Util.toCollectionColumns(CassandraUtil.changeCassandraColumnMapping(map)); + map = CassandraUtil.changeCassandraColumnMapping(map); return cassandraOperation.insertRecord( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), map, requestContext); } @@ -41,12 +41,11 @@ public Response update(RequestContext requestContext, String courseId, String ba Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); - Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Map attributeMap = new HashMap<>(); attributeMap.putAll(map); attributeMap.remove(JsonKey.COURSE_ID); attributeMap.remove(JsonKey.BATCH_ID); - attributeMap = Util.toCollectionColumns(CassandraUtil.changeCassandraColumnMapping(attributeMap)); + attributeMap = CassandraUtil.changeCassandraColumnMapping(attributeMap); return cassandraOperation.updateRecord( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), attributeMap, primaryKey, requestContext); } @@ -56,7 +55,6 @@ public CourseBatch readById(String courseId, String batchId, RequestContext requ Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); - Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Response courseBatchResult = cassandraOperation.getRecordByIdentifier( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), primaryKey, null, requestContext); @@ -78,7 +76,6 @@ public Map getCourseBatch(RequestContext requestContext, String Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); - Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Response courseBatchResult = cassandraOperation.getRecordByIdentifier( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), primaryKey, null, requestContext); @@ -99,7 +96,6 @@ public void addCertificateTemplateToCourseBatch( Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); - Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) cassandraOperation.updateAddMapRecord( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), @@ -115,7 +111,6 @@ public void removeCertificateTemplateFromCourseBatch( Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); - Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) cassandraOperation.updateRemoveMapRecord( courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/UserCoursesDaoImpl.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/UserCoursesDaoImpl.java index eb55dedf..f65bbf01 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/UserCoursesDaoImpl.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/UserCoursesDaoImpl.java @@ -35,7 +35,6 @@ public UserCourses read(RequestContext requestContext, String batchId, String us Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.BATCH_ID, batchId); primaryKey.put(JsonKey.USER_ID, userId); - Util.toCollectionColumns(primaryKey); // viewer: batchid->contextid (no-op when disabled) Response response = cassandraOperation.getRecordByIdentifier(KEYSPACE_NAME, TABLE_NAME, primaryKey, null, requestContext); List> userCoursesList = (List>) response.get(JsonKey.RESPONSE); @@ -55,7 +54,6 @@ public Response update(RequestContext requestContext, String batchId, String use Map primaryKey = new HashMap<>(); primaryKey.put(JsonKey.BATCH_ID, batchId); primaryKey.put(JsonKey.USER_ID, userId); - Util.toCollectionColumns(primaryKey); // viewer: batchid->contextid (no-op when disabled) Map updateList = new HashMap<>(); updateList.putAll(updateAttributes); updateList.remove(JsonKey.BATCH_ID); @@ -89,7 +87,6 @@ public Response updateV2(RequestContext requestContext, String userId, String co primaryKey.put(JsonKey.USER_ID, userId); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); - Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Map updateList = new HashMap<>(); updateList.putAll(updateAttributes); updateList.remove(JsonKey.BATCH_ID_KEY); @@ -104,7 +101,6 @@ public UserCourses read(RequestContext requestContext, String userId, String cou primaryKey.put(JsonKey.USER_ID, userId); primaryKey.put(JsonKey.COURSE_ID, courseId); primaryKey.put(JsonKey.BATCH_ID, batchId); - Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid, batchid->contextid (no-op when disabled) Response response = cassandraOperation.getRecordByIdentifier(KEYSPACE_NAME, USER_ENROLMENTS, primaryKey, null, requestContext); List> userCoursesList = (List>) response.get(JsonKey.RESPONSE); @@ -123,9 +119,7 @@ public List getBatchParticipants(RequestContext requestContext, String b Map queryMap = new HashMap<>(); queryMap.put(JsonKey.BATCH_ID, batchId); Response response = - // viewer: user_enrolments is indexed on collectionid (batchid->contextid has NO index in the - // generalised schema) — this by-batch query needs a contextid index added to work under viewer. - cassandraOperation.getRecordsByIndexedProperty(KEYSPACE_NAME, USER_ENROLMENTS, Util.VIEWER_ENABLED ? "contextid" : "batchid", batchId, requestContext); + cassandraOperation.getRecordsByIndexedProperty(KEYSPACE_NAME, USER_ENROLMENTS, "batchid", batchId, requestContext); /*cassandraOperation.getRecords( requestContext, KEYSPACE_NAME, USER_ENROLMENTS, queryMap, Arrays.asList(JsonKey.USER_ID, JsonKey.ACTIVE));*/ List> userCoursesList = @@ -147,7 +141,6 @@ public List> listEnrolments(RequestContext requestContext, S if(!CollectionUtils.isEmpty(courseIdList)){ primaryKey.put(JsonKey.COURSE_ID_KEY, courseIdList); } - Util.toCollectionColumns(primaryKey); // viewer: courseid->collectionid (no-op when disabled) Response response = cassandraOperation.getRecordByIdentifier(KEYSPACE_NAME, USER_ENROLMENTS, primaryKey, null, requestContext); List> userCoursesList = (List>) response.get(JsonKey.RESPONSE); if (CollectionUtils.isEmpty(userCoursesList)) { diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/search/SearchHandlerActor.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/search/SearchHandlerActor.java index 4cba52ad..bb7990e8 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/search/SearchHandlerActor.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/search/SearchHandlerActor.java @@ -120,15 +120,6 @@ public void onReceive(Request request) throws Throwable { if (EsType.courseBatch.getTypeName().equalsIgnoreCase(filterObjectType)) { List> courseBatchList = (List>) result.get(JsonKey.CONTENT); - // viewer.enabled: courseBatch docs are stored with collectionId/contextId; map back to the API - // contract (courseId/batchId) so clients and downstream (participants, status) are unchanged. - if (Util.VIEWER_ENABLED && CollectionUtils.isNotEmpty(courseBatchList)) { - courseBatchList.forEach(b -> { - if (b.containsKey("collectionId")) b.put(JsonKey.COURSE_ID, b.remove("collectionId")); - if (b.containsKey("contextId")) b.put(JsonKey.BATCH_ID, b.remove("contextId")); - }); - } - // Recompute status from dates for all search results to handle stale cached values if (CollectionUtils.isNotEmpty(courseBatchList)) { courseBatchList.forEach(batch -> CourseBatchUtil.enrichBatchStatusFromDates(batch)); diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java index 5d72a67e..7120f86e 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/CourseBatchUtil.java @@ -37,21 +37,8 @@ public class CourseBatchUtil { private CourseBatchUtil() {} - /** - * viewer.enabled: courseBatch ES docs use generalised identity fields (courseId->collectionId, - * batchId->contextId). Idempotent + no-op when viewer disabled, so it is safe to call on every - * courseBatch ES write path (create/update via esCourseMapping already renamed; cert/background - * writers pass a raw batch map and rely on this). - */ - public static void toEsCollectionFields(Map esMap) { - if (!Util.VIEWER_ENABLED || esMap == null) return; - if (esMap.containsKey(JsonKey.COURSE_ID)) esMap.put("collectionId", esMap.remove(JsonKey.COURSE_ID)); - if (esMap.containsKey(JsonKey.BATCH_ID)) esMap.put("contextId", esMap.remove(JsonKey.BATCH_ID)); - } - public static void syncCourseBatchForeground(RequestContext requestContext, String uniqueId, Map req) { logger.info(requestContext, "CourseBatchManagementActor: syncCourseBatchForeground called for course batch ID = " + uniqueId); - toEsCollectionFields(req); // unify all courseBatch ES writes on collectionId/contextId req.put(JsonKey.ID, uniqueId); req.put(JsonKey.IDENTIFIER, uniqueId); Future esResponseF = esUtil.save(ProjectUtil.EsType.courseBatch.getTypeName(), uniqueId, req, requestContext); @@ -67,7 +54,7 @@ public static Map validateCourseBatch(RequestContext requestCont ProjectCommonException.throwClientErrorException(ResponseCode.CLIENT_ERROR, "No such batchId exists"); } if (StringUtils.isNotBlank(courseId) - && !StringUtils.equals(courseId, (String) result.get(Util.VIEWER_ENABLED ? "collectionId" : JsonKey.COURSE_ID))) { + && !StringUtils.equals(courseId, (String) result.get(JsonKey.COURSE_ID))) { ProjectCommonException.throwClientErrorException(ResponseCode.CLIENT_ERROR, "batchId is not linked with courseId"); } return result; @@ -225,11 +212,6 @@ public static Map esCourseMapping(CourseBatch courseBatch, Strin }); esCourseMap.put(CourseJsonKey.CERTIFICATE_TEMPLATES_COLUMN, courseBatch.getCertTemplates()); - // viewer.enabled: courseBatch ES index uses generalised identity fields (courseId->collectionId, batchId->contextId) - if (Util.VIEWER_ENABLED) { - if (esCourseMap.containsKey(JsonKey.COURSE_ID)) esCourseMap.put("collectionId", esCourseMap.remove(JsonKey.COURSE_ID)); - if (esCourseMap.containsKey(JsonKey.BATCH_ID)) esCourseMap.put("contextId", esCourseMap.remove(JsonKey.BATCH_ID)); - } return esCourseMap; } diff --git a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/Util.java b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/Util.java index 6b0441d4..b9afd96f 100644 --- a/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/Util.java +++ b/modules/lms/course-mw/course-actors-common/src/main/java/org/sunbird/learner/util/Util.java @@ -99,29 +99,6 @@ private static Map manualMapCopy(Object mapObject) { initializeDBProperty(); } - /** - * When viewer_enabled the shared course tables are GENERALISED to a collection identity — - * courseid -> collectionid, batchid -> contextid on course_batch / user_enrolments / - * user_content_consumption. The legacy course DAOs still address these tables by courseid/batchid, - * so this remaps the identifier keys (camelCase JsonKey or physical column form) to the generalised - * column names in a query/attribute map. No-op when viewer is disabled (legacy schema). - */ - public static final boolean VIEWER_ENABLED = - Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")); - - public static Map toCollectionColumns(Map map) { - if (!VIEWER_ENABLED || map == null) return map; - moveKey(map, JsonKey.COURSE_ID, "collectionid"); // "courseId" - moveKey(map, JsonKey.COURSE_ID_KEY, "collectionid"); // "courseid" - moveKey(map, JsonKey.BATCH_ID, "contextid"); // "batchId" - moveKey(map, JsonKey.BATCH_ID_KEY, "contextid"); // "batchid" - return map; - } - - private static void moveKey(Map map, String from, String to) { - if (map.containsKey(from)) map.put(to, map.remove(from)); - } - private Util() {} /** This method will initialize the cassandra data base property */ diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala index f4644db4..b9068edd 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/AssessmentAuditRecorder.scala @@ -49,12 +49,9 @@ object AssessmentAuditRecorder { private def createRecordMap(m: util.Map[String, AnyRef], aid: String, uid: String, cid: String, ts: Long, ctx: RequestContext): util.Map[String, AnyRef] = { val rec = new util.HashMap[String, AnyRef]() - // assessment_aggregator identity columns gated by viewer_enabled (new names when migrated / viewer on, - // legacy names when off) — keeps the legacy assessment path writing the un-migrated schema. - val viewerEnabled = java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled")) rec.put("user_id", uid) - rec.put(if (viewerEnabled) "collection_id" else "course_id", m.get(JsonKey.COURSE_ID)) - rec.put(if (viewerEnabled) "context_id" else "batch_id", m.get(JsonKey.BATCH_ID)) + rec.put("course_id", m.get(JsonKey.COURSE_ID)) + rec.put("batch_id", m.get(JsonKey.BATCH_ID)) rec.put("content_id", cid) rec.put("attempt_id", aid) logger.info(ctx, s"AssessmentAuditRecorder: Recording attemptId=$aid with last_attempted_on=$ts") diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala index f68f5a74..8f7c3964 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/BaseEnrolmentActor.scala @@ -7,7 +7,6 @@ import org.sunbird.common.factory.EsClientFactory import org.sunbird.common.inf.ElasticSearchService import org.sunbird.keys.JsonKey import org.sunbird.common.ProjectUtil -import org.sunbird.learner.util.Util import org.sunbird.request.RequestContext import org.sunbird.dto.SearchDTO @@ -20,8 +19,7 @@ abstract class BaseEnrolmentActor extends BaseActor { def getBatches(requestContext: RequestContext, batchIds: java.util.List[String], requestedFields: java.util.List[String]): java.util.List[java.util.Map[String, AnyRef]] = { val dto = new SearchDTO dto.setLimit(batchIds.size()) - val batchField = if (Util.VIEWER_ENABLED) "contextId" else JsonKey.BATCH_ID - dto.getAdditionalProperties().put(JsonKey.FILTERS, new java.util.HashMap[String, AnyRef](){{ put(batchField, batchIds)}}) + dto.getAdditionalProperties().put(JsonKey.FILTERS, new java.util.HashMap[String, AnyRef](){{ put(JsonKey.BATCH_ID, batchIds)}}) if(CollectionUtils.isNotEmpty(requestedFields)) dto.setFields(requestedFields) val future = esService.search(dto, ProjectUtil.EsType.courseBatch.getTypeName, requestContext) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala index d9e5114d..85643b06 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala @@ -207,8 +207,7 @@ class ContentConsumptionActor @Inject() ( userContents.foreach(entry => { val userId = entry._1 if(validUserIds.contains(userId)) { - val courseId = if (entry._2.head.containsKey(JsonKey.COURSE_ID)) entry._2.head.getOrDefault(JsonKey.COURSE_ID, "").asInstanceOf[String] else entry._2.head.getOrDefault(JsonKey.COLLECTION_ID, "").asInstanceOf[String] - if(entry._2.head.containsKey(JsonKey.COLLECTION_ID)) entry._2.head.remove(JsonKey.COLLECTION_ID) + val courseId = entry._2.head.getOrDefault(JsonKey.COURSE_ID, "").asInstanceOf[String] val contentIds = entry._2.map(e => e.getOrDefault(JsonKey.CONTENT_ID, "").asInstanceOf[String]).distinct.asJava val existingContents = getContentsConsumption(userId, courseId, contentIds, batchId, requestContext).groupBy(x => x.get("contentId").asInstanceOf[String]).map(e => e._1 -> e._2.toList.head).toMap val contents:List[java.util.Map[String, AnyRef]] = entry._2.toList.map(inputContent => { @@ -508,12 +507,12 @@ class ContentConsumptionActor @Inject() ( val contentId = c.get(JsonKey.CONTENT_ID).asInstanceOf[String] try { val status = c.getOrDefault(JsonKey.STATUS, 0.asInstanceOf[AnyRef]).asInstanceOf[Number].intValue() - val collectionId = Option(c.get(JsonKey.COLLECTION_ID)).getOrElse(c.get(JsonKey.COURSE_ID)).asInstanceOf[String] + val courseId = c.get(JsonKey.COURSE_ID).asInstanceOf[String] val (op, api) = if (status >= 2) ("viewEnd", "/v1/view/end") else ("viewStart", "/v1/view/start") val body = new java.util.HashMap[String, AnyRef]() {{ put("contentId", contentId) - put("collectionId", collectionId) - put("contextId", c.get(JsonKey.BATCH_ID)) + put("courseId", courseId) + put("batchId", c.get(JsonKey.BATCH_ID)) put(JsonKey.USER_ID, userId) Option(c.get("progressdetails")).orElse(Option(c.get("progressDetails"))).foreach(pd => put("progressDetails", pd)) }} @@ -543,12 +542,12 @@ class ContentConsumptionActor @Inject() ( assessmentEvents.asScala.foreach(a => { val batchId = a.getOrDefault(JsonKey.BATCH_ID, "").asInstanceOf[String] try { - val collectionId = Option(a.get(JsonKey.COLLECTION_ID)).getOrElse(a.get(JsonKey.COURSE_ID)).asInstanceOf[String] + val courseId = a.get(JsonKey.COURSE_ID).asInstanceOf[String] val events = a.getOrDefault(JsonKey.ASSESSMENT_EVENTS_KEY, new java.util.ArrayList[java.util.Map[String, AnyRef]]()) val body = new java.util.HashMap[String, AnyRef]() {{ put("contentId", a.get(JsonKey.CONTENT_ID)) - put("collectionId", collectionId) - put("contextId", batchId) + put("courseId", courseId) + put("batchId", batchId) put(JsonKey.USER_ID, userId) put(JsonKey.ASSESSMENT_EVENTS, events) }} @@ -573,8 +572,8 @@ class ContentConsumptionActor @Inject() ( val token = originalRequest.getContext.get(JsonKey.X_AUTH_TOKEN).asInstanceOf[String] val body = new java.util.HashMap[String, AnyRef]() {{ put(JsonKey.USER_ID, userId) - put("collectionId", courseId) - put("contextId", batchId) + put("courseId", courseId) + put("batchId", batchId) if (CollectionUtils.isNotEmpty(contentIds)) put("contentId", contentIds) }} try viewerRead("view-consumption-actor", "/v1/view/read", "viewRead", body, token, ctx) @@ -650,9 +649,8 @@ class ContentConsumptionActor @Inject() ( val filters = new java.util.HashMap[String, AnyRef]() { { put("user_id", userId) - // gated by viewer_enabled: new names when the assessment table is migrated, legacy otherwise - put(if (isViewerEnabled) "collection_id" else "course_id", courseId) - put(if (isViewerEnabled) "context_id" else "batch_id", batchId) + put("course_id", courseId) + put("batch_id", batchId) put("content_id", contentId) } } diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index c91e1673..dcfd5f12 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -145,10 +145,9 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def list(request: Request): Unit = { val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] val courseIdList = request.get(JsonKey.COURSE_IDS).asInstanceOf[java.util.List[String]] - val useCache = isCacheEnabled && request.getContext.get("cache").asInstanceOf[Boolean] - logger.info(request.getRequestContext,"CourseEnrolmentActor :: list :: UserId = " + userId + " courseIdList=" + courseIdList + " isCacheEnabled=" + isCacheEnabled + " contextCacheFlag=" + request.getContext.get("cache") + " => useCache=" + useCache) + logger.info(request.getRequestContext,"CourseEnrolmentActor :: list :: UserId = " + userId) try{ - val response = if (useCache) + val response = if (isCacheEnabled && request.getContext.get("cache").asInstanceOf[Boolean]) getCachedEnrolmentList(userId, () => getEnrolmentList(request, userId, courseIdList)) else getEnrolmentList(request, userId, courseIdList) sender().tell(response, self) }catch { @@ -161,43 +160,27 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def getActiveEnrollments(userId: String, courseIdList: java.util.List[String], requestContext: RequestContext): java.util.List[java.util.Map[String, AnyRef]] = { val enrolments: java.util.List[java.util.Map[String, AnyRef]] = userCoursesDao.listEnrolments(requestContext, userId, courseIdList) - logger.info(requestContext, "getActiveEnrollments :: userId=" + userId + " viewerEnabled=" + Util.VIEWER_ENABLED + " rawEnrolments=" + (if (enrolments == null) "null" else enrolments.size.toString) + " firstRowKeys=" + (if (CollectionUtils.isNotEmpty(enrolments)) enrolments.get(0).keySet.toString else "[]")) if (CollectionUtils.isNotEmpty(enrolments)) { - // viewer.enabled: enrolment rows read back as collectionId/contextId (camelCase when the column-mapping - // properties are deployed, else lowercase collectionid/contextid); restore the API contract courseId/batchId. - if (Util.VIEWER_ENABLED) enrolments.forEach(e => { - if (e.containsKey("collectionId")) e.put(JsonKey.COURSE_ID, e.remove("collectionId")) - else if (e.containsKey("collectionid")) e.put(JsonKey.COURSE_ID, e.remove("collectionid")) - if (e.containsKey("contextId")) e.put(JsonKey.BATCH_ID, e.remove("contextId")) - else if (e.containsKey("contextid")) e.put(JsonKey.BATCH_ID, e.remove("contextid")) - }) - logger.info(requestContext, "getActiveEnrollments :: after viewer remap :: firstRow courseId=" + enrolments.get(0).get(JsonKey.COURSE_ID) + " batchId=" + enrolments.get(0).get(JsonKey.BATCH_ID) + " active=" + enrolments.get(0).get(JsonKey.ACTIVE)) val activeEnrolments = enrolments.filter(e => e.getOrDefault(JsonKey.ACTIVE, false.asInstanceOf[AnyRef]).asInstanceOf[Boolean]) - logger.info(requestContext, "getActiveEnrollments :: activeEnrolments(after active=true filter)=" + activeEnrolments.size) val sortedEnrolment = activeEnrolments.filter(ae => ae.get(JsonKey.COURSE_ENROLL_DATE)!=null).toList.sortBy(_.get(JsonKey.COURSE_ENROLL_DATE).asInstanceOf[Date])(Ordering[Date].reverse).toList val finalEnrolments = sortedEnrolment ++ activeEnrolments.filter(e => e.get(JsonKey.COURSE_ENROLL_DATE)==null).toList - logger.info(requestContext, "getActiveEnrollments :: finalEnrolments=" + finalEnrolments.size + " (limit=" + ProjectUtil.getConfigValue("enrollment_list_size") + ")") finalEnrolments.take(Integer.parseInt(ProjectUtil.getConfigValue("enrollment_list_size"))).toList.asJava } else { - logger.info(requestContext, "getActiveEnrollments :: no enrolments read from cassandra for userId=" + userId) new util.ArrayList[java.util.Map[String, AnyRef]]() } } def addCourseDetails(activeEnrolments: java.util.List[java.util.Map[String, AnyRef]], courseIds: java.util.List[String] , request:Request): java.util.List[java.util.Map[String, AnyRef]] = { val requestBody: String = prepareSearchRequest(courseIds, request) - logger.info(request.getRequestContext, "addCourseDetails :: courseIds=" + courseIds + " searchRequestBody=" + requestBody) val searchResult:java.util.Map[String, AnyRef] = ContentSearchUtil.searchContentSync(request.getRequestContext, request.getContext.getOrDefault(JsonKey.URL_QUERY_STRING,"").asInstanceOf[String], requestBody, request.get(JsonKey.HEADER).asInstanceOf[java.util.Map[String, String]]) - logger.info(request.getRequestContext, "addCourseDetails :: searchResult=" + (if (searchResult == null) "NULL(search call failed)" else "keys=" + searchResult.keySet)) val coursesList: java.util.List[java.util.Map[String, AnyRef]] = searchResult.getOrDefault(JsonKey.CONTENTS, new java.util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] val coursesMap = { if(CollectionUtils.isNotEmpty(coursesList)) { coursesList.map(ev => ev.get(JsonKey.IDENTIFIER).asInstanceOf[String] -> ev).toMap } else Map() } - logger.info(request.getRequestContext, "addCourseDetails :: coursesList=" + (if (coursesList == null) "null" else coursesList.size.toString) + " coursesMapKeys=" + coursesMap.keySet + " enrolmentCourseIds=" + activeEnrolments.map(e => e.get(JsonKey.COURSE_ID)).mkString("[", ",", "]")) - val withCourse = activeEnrolments.filter(enrolment => coursesMap.containsKey(enrolment.get(JsonKey.COURSE_ID))).map(enrolment => { + activeEnrolments.filter(enrolment => coursesMap.containsKey(enrolment.get(JsonKey.COURSE_ID))).map(enrolment => { val courseContent = coursesMap.get(enrolment.get(JsonKey.COURSE_ID)) enrolment.put(JsonKey.COURSE_NAME, courseContent.get(JsonKey.NAME)) enrolment.put(JsonKey.DESCRIPTION, courseContent.get(JsonKey.DESCRIPTION)) @@ -208,8 +191,6 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c enrolment.put(JsonKey.CONTENT, courseContent) enrolment }).toList.asJava - logger.info(request.getRequestContext, "addCourseDetails :: keptAfterCourseFilter=" + withCourse.size) - withCourse } def prepareSearchRequest(courseIds: java.util.List[String], request: Request): String = { @@ -232,14 +213,8 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def addBatchDetails(enrolmentList: util.List[util.Map[String, AnyRef]], request: Request): util.List[util.Map[String, AnyRef]] = { val batchIds:java.util.List[String] = enrolmentList.map(e => e.getOrDefault(JsonKey.BATCH_ID, "").asInstanceOf[String]).distinct.filter(id => StringUtils.isNotBlank(id)).toList.asJava val batchDetails = searchBatchDetails(batchIds, request) - logger.info(request.getRequestContext, "addBatchDetails :: enrolmentListSize=" + enrolmentList.size + " batchIds=" + batchIds + " batchDetailsFound=" + (if (batchDetails == null) "null" else batchDetails.size.toString)) if(CollectionUtils.isNotEmpty(batchDetails)){ batchDetails.foreach(batch => CourseBatchUtil.enrichBatchStatusFromDates(batch)) - // viewer.enabled: batch docs are stored with collectionId/contextId; map back to the API contract courseId/batchId - if (Util.VIEWER_ENABLED) batchDetails.foreach { b => - if (b.containsKey("collectionId")) b.put(JsonKey.COURSE_ID, b.remove("collectionId")) - if (b.containsKey("contextId")) b.put(JsonKey.BATCH_ID, b.remove("contextId")) - } val batchMap = batchDetails.map(b => b.get(JsonKey.BATCH_ID).asInstanceOf[String] -> b).toMap enrolmentList.map(enrolment => { enrolment.put(JsonKey.BATCH, batchMap.getOrElse(enrolment.get(JsonKey.BATCH_ID).asInstanceOf[String], new java.util.HashMap[String, AnyRef]())) @@ -290,8 +265,7 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c def upsertEnrollment(userId: String, courseId: String, batchId: String, data: java.util.Map[String, AnyRef], isNew: Boolean, requestContext: RequestContext): Unit = { val dataMap = CassandraUtil.changeCassandraColumnMapping(data) if(isNew) { - // viewer: remap courseid->collectionid, batchid->contextid on the insert row (no-op when disabled). - userCoursesDao.insertV2(requestContext, Util.toCollectionColumns(dataMap)) + userCoursesDao.insertV2(requestContext, dataMap) } else { userCoursesDao.updateV2(requestContext, userId, courseId, batchId, dataMap) } @@ -368,10 +342,8 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c val key = getCacheKey(userId) val responseString = cacheUtil.get(key) if (StringUtils.isNotBlank(responseString)) { - logger.info(null.asInstanceOf[RequestContext], "getCachedEnrolmentList :: CACHE HIT key=" + key + " (serving cached response, len=" + responseString.length + ")") JsonUtil.deserialize(responseString, classOf[Response]) } else { - logger.info(null.asInstanceOf[RequestContext], "getCachedEnrolmentList :: CACHE MISS key=" + key + " -> querying cassandra") val response = handleEmptyCache() val responseString = JsonUtil.serialize(response) cacheUtil.set(key, responseString, ttl) @@ -396,7 +368,6 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c val resp: Response = new Response() val sortedEnrolment = enrolments.filter(ae => ae.get("lastContentAccessTime")!=null).toList.sortBy(_.get("lastContentAccessTime").asInstanceOf[Date])(Ordering[Date].reverse).toList val finalEnrolments = sortedEnrolment ++ enrolments.asScala.filter(e => e.get("lastContentAccessTime")==null).toList - logger.info(request.getRequestContext, "getEnrolmentList :: userId=" + userId + " FINAL courses returned=" + finalEnrolments.size) resp.put(JsonKey.COURSES, finalEnrolments.asJava) resp } diff --git a/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java b/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java index 279cfff2..467c6322 100644 --- a/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java +++ b/modules/lms/service/app/controllers/coursemanagement/CourseBatchController.java @@ -7,7 +7,6 @@ import controllers.coursemanagement.validator.CourseBatchRequestValidator; import org.sunbird.operations.lms.ActorOperations; import org.sunbird.keys.JsonKey; -import org.sunbird.learner.util.Util; import org.sunbird.common.ProjectUtil.EsType; import org.sunbird.request.Request; import play.mvc.Http; @@ -40,8 +39,6 @@ public CompletionStage createBatch(Http.Request httpRequest) { httpRequest.body().asJson(), (request) -> { Request req = (Request) request; - String courseId = req.getRequest().containsKey(JsonKey.COURSE_ID) ? JsonKey.COURSE_ID : JsonKey.COLLECTION_ID; - req.getRequest().put(JsonKey.COURSE_ID, req.getRequest().get(courseId)); new CourseBatchRequestValidator().validateCreateCourseBatchRequest(req); return null; }, @@ -70,8 +67,6 @@ public CompletionStage updateBatch(Http.Request httpRequest) { httpRequest.body().asJson(), (request) -> { Request req = (Request) request; - String courseId = req.getRequest().containsKey(JsonKey.COURSE_ID) ? JsonKey.COURSE_ID : JsonKey.COLLECTION_ID; - req.getRequest().put(JsonKey.COURSE_ID, req.getRequest().get(courseId)); new CourseBatchRequestValidator().validateUpdateCourseBatchRequest(req); return null; }, @@ -100,11 +95,6 @@ public CompletionStage search(Http.Request httpRequest) { && reqObj.getRequest().get(JsonKey.FILTERS) != null && reqObj.getRequest().get(JsonKey.FILTERS) instanceof Map) { Map f = (Map) reqObj.getRequest().get(JsonKey.FILTERS); - // viewer.enabled: courseBatch ES index uses generalised identity fields; translate client filter keys - if (Util.VIEWER_ENABLED) { - if (f.containsKey(JsonKey.COURSE_ID)) f.put("collectionId", f.remove(JsonKey.COURSE_ID)); - if (f.containsKey(JsonKey.BATCH_ID)) f.put("contextId", f.remove(JsonKey.BATCH_ID)); - } f.put(JsonKey.OBJECT_TYPE, esObjectType); } else { Map filtermap = new HashMap<>(); diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala index 76623ec8..60d782ad 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala @@ -28,8 +28,8 @@ import scala.collection.JavaConverters._ * viewUpdate -> merge only if row exists; revisits (already status 2) ignored. * viewEnd -> status 2 + completed time; then async aggregation (fire-and-forget tell). * - * ucc PK (viewer schema §2): (userid, collectionid, contextid, contentid). - * No collection context -> collectionid = contextid = contentid. + * ucc PK (viewer schema §2): (userid, courseid, batchid, contentid). + * No collection context -> courseid = batchid = contentid. */ class ViewConsumptionActor @Inject() ( @Named("viewer-aggregator-actor") viewerAggregatorActor: ActorRef @@ -66,14 +66,14 @@ class ViewConsumptionActor @Inject() ( * like a completed content: mark ucc status=2 and run the same sync rollup as viewEnd, so the * assessment leaf counts toward collection completion. Score aggregates (score:cid/max_score:cid) * land in user_activity_agg via putAll append; the rollup's completion agg uses different keys → - * they coexist. Request: userId, collectionId?, contextId?, contentId, assessments[] (assess events). + * they coexist. Request: userId, courseId?, batchId?, contentId, assessments[] (assess events). */ private def viewAssess(request: Request): Unit = { val ctx = request.getRequestContext val key = viewKey(request) val userId = key.get("userid").asInstanceOf[String] - val collectionId = key.get("collectionid").asInstanceOf[String] - val contextId = key.get("contextid").asInstanceOf[String] + val courseId = key.get("courseid").asInstanceOf[String] + val batchId = key.get("batchid").asInstanceOf[String] val contentId = key.get("contentid").asInstanceOf[String] val eventsRaw = Option(request.get(JsonKey.ASSESSMENT_EVENTS)).orElse(Option(request.get(JsonKey.EVENTS))) @@ -87,13 +87,13 @@ class ViewConsumptionActor @Inject() ( .getOrElse(java.util.UUID.randomUUID().toString) // no client attemptId -> a fresh attempt (avoids hashCode collisions overwriting a prior attempt) val unique = assessmentService.getUniqueQuestions(events) val metrics = assessmentService.computeScoreMetrics(unique) - val result = AssessmentResult(attemptId, userId, collectionId, contextId, contentId, + val result = AssessmentResult(attemptId, userId, courseId, batchId, contentId, metrics.totalScore, metrics.totalMaxScore, metrics.grandTotal, metrics.questions, System.currentTimeMillis(), ts) assessmentCassandra.saveAssessment(result, ctx) // best-score across all attempts -> user_activity_agg (reuse legacy aggregation) - val stored = assessmentCassandra.getUserAssessments(userId, collectionId, contextId, contentId, ctx) - val agg = assessmentService.computeUserAggregates(userId, collectionId, contextId, stored) - assessmentCassandra.updateUserActivity(userId, collectionId, contextId, agg, ctx) + val stored = assessmentCassandra.getUserAssessments(userId, courseId, batchId, contentId, ctx) + val agg = assessmentService.computeUserAggregates(userId, courseId, batchId, stored) + assessmentCassandra.updateUserActivity(userId, courseId, batchId, agg, ctx) } else { logger.warn(ctx, s"viewAssess: no assessment events for userId=$userId contentId=$contentId; marking complete only", null) } @@ -111,13 +111,13 @@ class ViewConsumptionActor @Inject() ( /** * /v1/assessment/read (api.assessment.read). Best score / max score per content from - * assessment_aggregator (reuse getUserAssessments). Request: userId, contentId[] , collectionId?, contextId?. + * assessment_aggregator (reuse getUserAssessments). Request: userId, contentId[] , courseId?, batchId?. */ private def assessmentRead(request: Request): Unit = { val ctx = request.getRequestContext val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] - val collectionId = ViewerRequestKeys.collectionId(request).orNull - val contextId = ViewerRequestKeys.contextId(request).orNull + val courseId = ViewerRequestKeys.courseId(request).orNull + val batchId = ViewerRequestKeys.batchId(request).orNull val contentIds: List[String] = request.get("contentId") match { case l: util.List[_] => l.asScala.map(_.asInstanceOf[String]).toList case s: String if StringUtils.isNotBlank(s) => List(s) @@ -125,7 +125,7 @@ class ViewConsumptionActor @Inject() ( } val contents = new util.ArrayList[util.Map[String, AnyRef]]() contentIds.foreach { cid => - val stored = assessmentCassandra.getUserAssessments(userId, collectionId, contextId, cid, ctx) + val stored = assessmentCassandra.getUserAssessments(userId, courseId, batchId, cid, ctx) if (stored.nonEmpty) { val best = stored.maxBy(_.totalScore) val m = new util.HashMap[String, AnyRef]() @@ -137,21 +137,21 @@ class ViewConsumptionActor @Inject() ( } val out = new Response() out.put(JsonKey.USER_ID, userId) - out.put("collectionId", collectionId) - out.put("contextId", contextId) + out.put("courseId", courseId) + out.put("batchId", batchId) out.put("contents", contents) sender().tell(out, self) } - /** Raw ucc rows for a user's content(s) under a collection. context=all -> ignore contextid. */ + /** Raw ucc rows for a user's content(s) under a collection. context=all -> ignore batchid. */ private def viewRead(request: Request): Unit = { val ctx = request.getRequestContext val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] val allContexts = "all".equalsIgnoreCase(request.get("context").asInstanceOf[String]) val filters = new util.HashMap[String, AnyRef]() filters.put("userid", userId) - ViewerRequestKeys.collectionId(request).foreach(c => filters.put("collectionid", c)) - if (!allContexts) ViewerRequestKeys.contextId(request).foreach(c => filters.put("contextid", c)) + ViewerRequestKeys.courseId(request).foreach(c => filters.put("courseid", c)) + if (!allContexts) ViewerRequestKeys.batchId(request).foreach(c => filters.put("batchid", c)) val contentIds = request.get("contentId") match { case l: util.List[_] => l.asScala.map(_.asInstanceOf[String]).asJava case s: String if StringUtils.isNotBlank(s) => util.Arrays.asList(s) @@ -217,17 +217,17 @@ class ViewConsumptionActor @Inject() ( aggRequest.setOperation("aggregate") aggRequest.setRequestContext(ctx) aggRequest.put(JsonKey.USER_ID, key.get("userid")) - aggRequest.put("collectionId", key.get("collectionid")) - aggRequest.put("contextId", key.get("contextid")) + aggRequest.put("courseId", key.get("courseid")) + aggRequest.put("batchId", key.get("batchid")) // Async, fire-and-forget: the rollup + LP progression run in the background on the aggregator // (per-user serialized). The hot path does not wait for it — the change from before is ask -> tell. viewerAggregatorActor.tell(aggRequest, ActorRef.noSender) } /** - * Build the ucc primary key (live column names userid, collectionid, contextid, contentid). - * Backward-compatible request keys: collectionId (else legacy courseId), contextId (else legacy - * batchId). No collection ctx -> collectionid = contextid = contentId. + * Build the ucc primary key (live column names userid, courseid, batchid, contentid). + * Backward-compatible request keys: courseId (else legacy courseId), batchId (else legacy + * batchId). No collection ctx -> courseid = batchid = contentId. */ private def viewKey(request: Request): util.HashMap[String, AnyRef] = { // explicit userId (internal delegation) else requestedFor/requestedBy (from token on direct API calls) @@ -235,12 +235,12 @@ class ViewConsumptionActor @Inject() ( .orElse(Option(request.get(JsonKey.REQUESTED_FOR).asInstanceOf[String]).filter(StringUtils.isNotBlank)) .getOrElse(request.get(JsonKey.REQUESTED_BY).asInstanceOf[String]) val contentId = ViewerRequestKeys.contentId(request) - val collectionId = ViewerRequestKeys.collectionId(request).getOrElse(contentId) - val contextId = ViewerRequestKeys.contextId(request).getOrElse(contentId) + val courseId = ViewerRequestKeys.courseId(request).getOrElse(contentId) + val batchId = ViewerRequestKeys.batchId(request).getOrElse(contentId) val key = new util.HashMap[String, AnyRef]() key.put("userid", userId) - key.put("collectionid", collectionId) - key.put("contextid", contextId) + key.put("courseid", courseId) + key.put("batchid", batchId) key.put("contentid", contentId) key } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index 7f04a052..e0f4c0b1 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -20,7 +20,7 @@ import scala.collection.JavaConverters._ * Sync, in-request recursive rollup for the viewer module. Invoked from viewEnd (per-userId serialized). * * REUSE: all aggregation math is ActivityAggregateUtil (same calls ActivityAggregatorActor uses). - * The util treats collectionId as the activity_id slot and batchId as the context slot — it does not + * The util treats courseId as the activity_id slot and batchId as the context slot — it does not * care about the names. Viewer deltas vs ActivityAggregatorActor: * - optionality is PER-USER: `optional_nodes` from user_enrolments (NOT hierarchy getOptionalNodes). * required = collectionLeafNodes.diff(userOpt) at leaf AND every ancestor level. @@ -57,55 +57,55 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { private def aggregate(request: Request): Unit = { val ctx = request.getRequestContext val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] - // accept collectionId (else legacy courseId) / contextId (else legacy batchId) - val collectionId = ViewerRequestKeys.collectionId(request).orNull - val batchId = ViewerRequestKeys.contextId(request).orNull - if (userId == null || collectionId == null) { - logger.warn(ctx, s"ViewerAggregatorActor: missing userId/collectionId, skipping", null) + // accept courseId (else legacy courseId) / batchId (else legacy batchId) + val courseId = ViewerRequestKeys.courseId(request).orNull + val batchId = ViewerRequestKeys.batchId(request).orNull + if (userId == null || courseId == null) { + logger.warn(ctx, s"ViewerAggregatorActor: missing userId/courseId, skipping", null) return } // trackablenodes non-empty => this root is a Learning Path (structural detection; §Step 2/5). - val trackable = hierarchyRelationsUtil.getTrackableNodes(collectionId, ctx) + val trackable = hierarchyRelationsUtil.getTrackableNodes(courseId, ctx) // 1. Read this user's consumption for the (root) collection+context from viewer ucc, build status map - val rows = readConsumption(userId, collectionId, batchId, ctx) + val rows = readConsumption(userId, courseId, batchId, ctx) if (CollectionUtils.isEmpty(rows)) { // No consumption yet. For an LP, still advance (bootstrap: open the first required course). - if (trackable.nonEmpty) advanceLp(userId, collectionId, batchId, trackable, ctx) - else logger.info(ctx, s"ViewerAggregatorActor: no consumption for userId=$userId collectionId=$collectionId") + if (trackable.nonEmpty) advanceLp(userId, courseId, batchId, trackable, ctx) + else logger.info(ctx, s"ViewerAggregatorActor: no consumption for userId=$userId courseId=$courseId") return } val contentStatusMap: Map[String, ContentStatus] = activityAggUtil.getContentStatusFromContents(rows) - val uc = UserContentConsumption(userId, batchId, collectionId, contentStatusMap) + val uc = UserContentConsumption(userId, batchId, courseId, contentStatusMap) // 2. Per-learner optional COURSES from user_enrolments.optional_nodes (LP policy; course-level). - val perLearnerOptionalCourses: List[String] = readOptionalNodes(userId, collectionId, batchId, ctx) + val perLearnerOptionalCourses: List[String] = readOptionalNodes(userId, courseId, batchId, ctx) // 3. Root leaves + the tree's nodes (via ancestors) — needed before computing effectiveOptional. - val leafNodes = hierarchyRelationsUtil.getLeafNodes(collectionId, collectionId, ctx) + val leafNodes = hierarchyRelationsUtil.getLeafNodes(courseId, courseId, ctx) if (leafNodes.isEmpty) { - logger.warn(ctx, s"ViewerAggregatorActor: no leafNodes for collectionId=$collectionId; is hierarchy_relations published?", null) + logger.warn(ctx, s"ViewerAggregatorActor: no leafNodes for courseId=$courseId; is hierarchy_relations published?", null) return } val ancestors: Map[String, List[String]] = uc.contents.map { case (contentId, content) => - (contentId, hierarchyRelationsUtil.getAncestors(collectionId, content.contentId, ctx)) + (contentId, hierarchyRelationsUtil.getAncestors(courseId, content.contentId, ctx)) }.toMap - val childCollections = ancestors.values.flatten.filter(_ != collectionId).toList.distinct + val childCollections = ancestors.values.flatten.filter(_ != courseId).toList.distinct // effectiveOptional LEAVES (§5.1) = author-marked hierarchy `optionalnodes` (content-level, all nodes) // ∪ leaves of per-learner optional courses (course-level, expanded to leaves so the leaf-vs-leaf diff works). - val treeNodes = collectionId :: childCollections - val hierarchyOptionalLeaves = treeNodes.flatMap(n => hierarchyRelationsUtil.getOptionalNodes(collectionId, n, ctx)).distinct - val optionalCourseLeaves = perLearnerOptionalCourses.flatMap(c => hierarchyRelationsUtil.getLeafNodes(collectionId, c, ctx)).distinct + val treeNodes = courseId :: childCollections + val hierarchyOptionalLeaves = treeNodes.flatMap(n => hierarchyRelationsUtil.getOptionalNodes(courseId, n, ctx)).distinct + val optionalCourseLeaves = perLearnerOptionalCourses.flatMap(c => hierarchyRelationsUtil.getLeafNodes(courseId, c, ctx)).distinct val effectiveOptional: List[String] = (hierarchyOptionalLeaves ++ optionalCourseLeaves).distinct // 4. Aggregates: root + every ancestor node; required per node = its leafNodes − effectiveOptional. val courseAgg = activityAggUtil.computeCourseActivityAgg(uc, leafNodes, effectiveOptional, ctx) val collectionsWithLeafNodes: Map[String, List[String]] = childCollections.map { col => - (col, hierarchyRelationsUtil.getLeafNodes(collectionId, col, ctx).diff(effectiveOptional)) + (col, hierarchyRelationsUtil.getLeafNodes(courseId, col, ctx).diff(effectiveOptional)) }.toMap - val moduleAggs = activityAggUtil.computeModuleActivityAgg(uc, collectionId, ancestors, collectionsWithLeafNodes, ctx) + val moduleAggs = activityAggUtil.computeModuleActivityAgg(uc, courseId, ancestors, collectionsWithLeafNodes, ctx) val allAggs: List[UserEnrolmentAgg] = courseAgg.toList ++ moduleAggs @@ -114,16 +114,16 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // 6. Per-node progress: nodeId -> (completedCount, requiredLeaves) for root + every trackable ancestor. val nodeProgress = scala.collection.mutable.LinkedHashMap[String, (Int, List[String])]() - courseAgg.foreach(a => nodeProgress(collectionId) = (completedCountOf(a), leafNodes.diff(effectiveOptional))) + courseAgg.foreach(a => nodeProgress(courseId) = (completedCountOf(a), leafNodes.diff(effectiveOptional))) moduleAggs.foreach(a => nodeProgress(a.activityAgg.activity_id) = (completedCountOf(a), collectionsWithLeafNodes.getOrElse(a.activityAgg.activity_id, Nil))) // 7. Update user_enrolments status for EVERY enrolled node in this tree (approach #1: key off the // child enrolment rows that already exist; root included). Cert fires once, on transition to complete. - writeAllNodeEnrolments(userId, collectionId, batchId, nodeProgress.toMap, contentStatusMap, ctx) + writeAllNodeEnrolments(userId, courseId, batchId, nodeProgress.toMap, contentStatusMap, ctx) // 8. LP progression (only when this root is an LP): optionality once, open next course(s), credit at completion. - if (trackable.nonEmpty) advanceLp(userId, collectionId, batchId, trackable, ctx) + if (trackable.nonEmpty) advanceLp(userId, courseId, batchId, trackable, ctx) } private def completedCountOf(a: UserEnrolmentAgg): Int = @@ -195,16 +195,16 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { private def isComplete(userId: String, courseId: String, rootBatchId: String, ctx: RequestContext): Boolean = enrolStatus(userId, courseId, rootBatchId + ":" + courseId, ctx).contains(2) - private def enrolStatus(userId: String, collectionId: String, contextId: String, ctx: RequestContext): Option[Int] = { - val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("collectionid", collectionId); put("contextid", contextId) }} + private def enrolStatus(userId: String, courseId: String, batchId: String, ctx: RequestContext): Option[Int] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", courseId); put("batchid", batchId) }} val rows = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] if (CollectionUtils.isNotEmpty(rows)) Option(rows.get(0).get("status")).map(_.asInstanceOf[Number].intValue()) else None } - private def isEnrolled(userId: String, collectionId: String, contextId: String, ctx: RequestContext): Boolean = - enrolStatus(userId, collectionId, contextId, ctx).isDefined + private def isEnrolled(userId: String, courseId: String, batchId: String, ctx: RequestContext): Boolean = + enrolStatus(userId, courseId, batchId, ctx).isDefined /** * Internal (system-driven) enrol via the ProgressionEnroller gateway — the FULL enrol op (`doEnrol`), @@ -214,12 +214,12 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { */ private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) // default monolith - private def internalEnrol(userId: String, collectionId: String, contextId: String, ctx: RequestContext): Unit = { + private def internalEnrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = { if (isMonolith) { val req = new Request() req.setRequestContext(ctx) req.setOperation("systemEnrol") - req.put(JsonKey.USER_ID, userId); req.put(JsonKey.COURSE_ID, collectionId); req.put(JsonKey.BATCH_ID, contextId) + req.put(JsonKey.USER_ID, userId); req.put(JsonKey.COURSE_ID, courseId); req.put(JsonKey.BATCH_ID, batchId) // VERIFY-ON-DEPLOY: bound path of the enrolment actor in the monolith actor system. val path = Option(ProjectUtil.getConfigValue("enrolment_actor_path")).filter(_.nonEmpty).getOrElse("/user/course-enrolment-actor") // noSender: fire-and-forget; the enrol actor's success reply must NOT bounce back to this actor @@ -229,7 +229,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // DISTRIBUTED: call the enrolment service over HTTP (full enrol op). // VERIFY-ON-DEPLOY: use a system-enrol endpoint (not the public one that fans out/notifies) + forward auth token. val base = Option(ProjectUtil.getConfigValue("enrolment_service_base_url")).filter(_.nonEmpty).getOrElse("http://lern-service:9000") - val body = s"""{"request":{"userId":"$userId","courseId":"$collectionId","batchId":"$contextId"}}""" + val body = s"""{"request":{"userId":"$userId","courseId":"$courseId","batchId":"$batchId"}}""" val headers = new util.HashMap[String, String]() {{ put("Content-Type", "application/json") // System-driven enrol: authenticate with the configured system token (else 401 in distributed mode). @@ -238,11 +238,11 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { }} HttpClientUtil.post(base + "/v1/course/enroll", body, headers, ctx) } - logger.info(ctx, s"ViewerAggregatorActor: system-enrol requested course=$collectionId ctx=$contextId user=$userId mode=${ProjectUtil.getConfigValue("deployment_mode")}") + logger.info(ctx, s"ViewerAggregatorActor: system-enrol requested course=$courseId ctx=$batchId user=$userId mode=${ProjectUtil.getConfigValue("deployment_mode")}") } private def writeOptionalNodes(userId: String, rootId: String, batchId: String, optional: Set[String], ctx: RequestContext): Unit = { - val selectMap = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("collectionid", rootId); put("contextid", batchId) }} + val selectMap = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", rootId); put("batchid", batchId) }} val updateMap = new util.HashMap[String, AnyRef]() {{ put("optional_nodes", optional.asJava) }} cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) ViewerAggregatorActor.markOptionalityComputed(userId, rootId, batchId) // remember empty results too (no DB column) @@ -283,7 +283,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { /** * Approach #1: update user_enrolments status for every node in this tree that has an enrolment row. - * Matches each row on collectionid ∈ tree AND this LP's contextid (§4: standalone enrolments untouched). + * Matches each row on courseid ∈ tree AND this LP's batchid (§4: standalone enrolments untouched). * Cert fires once, only on the transition to complete (status != 2 -> 2). */ private def writeAllNodeEnrolments(userId: String, rootId: String, batchId: String, @@ -295,9 +295,9 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) .asInstanceOf[util.List[util.Map[String, AnyRef]]] enrolRows.asScala.foreach { row => - val nodeId = Option(row.get("collectionid")).map(_.toString).orNull - val nodeCtx = Option(row.get("contextid")).map(_.toString).orNull - // This LP only (root=batchId, child=batchId:childId); a standalone enrolment's contextid differs (§4). + val nodeId = Option(row.get("courseid")).map(_.toString).orNull + val nodeCtx = Option(row.get("batchid")).map(_.toString).orNull + // This LP only (root=batchId, child=batchId:childId); a standalone enrolment's batchid differs (§4). val expectedCtx = if (nodeId == rootId) batchId else batchId + ":" + nodeId nodeProgress.get(nodeId).filter(_ => nodeCtx == expectedCtx).foreach { case (completedCount, requiredLeaves) => val required = requiredLeaves.size @@ -306,7 +306,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { val currentStatus = Option(row.get("status")).map(_.asInstanceOf[Number].intValue()).getOrElse(0) val nodeContentStatus = requiredLeaves.flatMap(l => contentStatusMap.get(l).map(cs => l -> Integer.valueOf(cs.status))).toMap val selectMap = new util.HashMap[String, AnyRef]() {{ - put("userid", userId); put("collectionid", nodeId); put("contextid", nodeCtx) + put("userid", userId); put("courseid", nodeId); put("batchid", nodeCtx) }} val updateMap = new util.HashMap[String, AnyRef]() {{ put("progress", Integer.valueOf(completedCount)) @@ -317,7 +317,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { }} cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) if (status == 2 && currentStatus != 2) { - logger.info(ctx, s"ViewerAggregatorActor: node completed userId=$userId collectionId=$nodeId; issuing cert") + logger.info(ctx, s"ViewerAggregatorActor: node completed userId=$userId courseId=$nodeId; issuing cert") certificateUtil.publishCertificateIssueEvent(userId, nodeId, nodeCtx, ctx) } } @@ -326,15 +326,15 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { /** * Read this user's ucc rows for the collection, scoped to the context. - * (userid, collectionid, contextid) is a clustering-prefix slice on PK - * (userid, collectionid, contextid, contentid) -> efficient, no scan. - * contextId omitted only when absent (no-context viewer), falling back to collection-wide read. + * (userid, courseid, batchid) is a clustering-prefix slice on PK + * (userid, courseid, batchid, contentid) -> efficient, no scan. + * batchId omitted only when absent (no-context viewer), falling back to collection-wide read. */ - private def readConsumption(userId: String, collectionId: String, contextId: String, ctx: RequestContext): util.List[util.Map[String, AnyRef]] = { + private def readConsumption(userId: String, courseId: String, batchId: String, ctx: RequestContext): util.List[util.Map[String, AnyRef]] = { val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) - put("collectionid", collectionId) - if (contextId != null) put("contextid", contextId) + put("courseid", courseId) + if (batchId != null) put("batchid", batchId) }} val response = cassandraOperation.getRecords(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) @@ -343,11 +343,11 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { } /** Per-user optional_nodes from user_enrolments (empty for strict policy). */ - private def readOptionalNodes(userId: String, collectionId: String, batchId: String, ctx: RequestContext): List[String] = { + private def readOptionalNodes(userId: String, courseId: String, batchId: String, ctx: RequestContext): List[String] = { val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) - put("collectionid", collectionId) - put("contextid", batchId) + put("courseid", courseId) + put("batchid", batchId) }} val response = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala index 6ce1573e..e73fc6b2 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerRequestKeys.scala @@ -4,7 +4,7 @@ import org.apache.commons.lang3.StringUtils import org.sunbird.request.Request /** - * Canonical viewer request keys — the viewer contract is collectionId / contextId / contentId only. + * Canonical viewer request keys — the viewer contract is courseId / batchId / contentId only. * Legacy courseId/batchId resolution is the caller's job (the content-consumption adapter maps them * before dispatching), so no fallback lives here. Null/blank-safe extraction in one place. */ @@ -13,7 +13,7 @@ object ViewerRequestKeys { private def value(request: Request, key: String): Option[String] = Option(request.get(key)).collect { case s: String if StringUtils.isNotBlank(s) => s } - def collectionId(request: Request): Option[String] = value(request, "collectionId") - def contextId(request: Request): Option[String] = value(request, "contextId") + def courseId(request: Request): Option[String] = value(request, "courseId") + def batchId(request: Request): Option[String] = value(request, "batchId") def contentId(request: Request): String = value(request, "contentId").orNull } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala index ca4a7b0a..f3858c14 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala @@ -36,19 +36,19 @@ class ViewerSummaryActor extends BaseEnrolmentActor { } } - /** Per-enrolment progress/status from user_enrolments (identified by collectionId). */ + /** Per-enrolment progress/status from user_enrolments (identified by courseId). */ private def summaryRead(request: Request): Unit = { val ctx = request.getRequestContext val userId = request.get(JsonKey.USER_ID).asInstanceOf[String] - val collectionId = ViewerRequestKeys.collectionId(request).orNull - val batchId = ViewerRequestKeys.contextId(request).orNull + val courseId = ViewerRequestKeys.courseId(request).orNull + val batchId = ViewerRequestKeys.batchId(request).orNull - // Identified by collectionId (courseid). user_enrolments carries progress/status/completionpercentage + // Identified by courseId (courseid). user_enrolments carries progress/status/completionpercentage // + per-content contentstatus for the enrolment — no activity_type needed. val enrolFilters = new util.HashMap[String, AnyRef]() enrolFilters.put("userid", userId) - if (StringUtils.isNotBlank(collectionId)) enrolFilters.put("collectionid", collectionId) - if (StringUtils.isNotBlank(batchId)) enrolFilters.put("contextid", batchId) + if (StringUtils.isNotBlank(courseId)) enrolFilters.put("courseid", courseId) + if (StringUtils.isNotBlank(batchId)) enrolFilters.put("batchid", batchId) val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, enrolFilters, ctx) val response = new Response @@ -89,7 +89,7 @@ class ViewerSummaryActor extends BaseEnrolmentActor { sender().tell(response, self) } - private val csvCols = List("collectionid", "contextid", "progress", "status", "completionpercentage", "completedon") + private val csvCols = List("courseid", "batchid", "progress", "status", "completionpercentage", "completedon") private def toCsv(rows: util.List[util.Map[String, AnyRef]]): String = { val sb = new StringBuilder(csvCols.mkString(",")).append("\n") rows.asScala.foreach { r => @@ -103,25 +103,25 @@ class ViewerSummaryActor extends BaseEnrolmentActor { val ctx = request.getRequestContext val userId = Option(request.get(JsonKey.USER_ID).asInstanceOf[String]) .getOrElse(request.get("userId").asInstanceOf[String]) - val collectionId = ViewerRequestKeys.collectionId(request).orNull - val batchId = ViewerRequestKeys.contextId(request).orNull + val courseId = ViewerRequestKeys.courseId(request).orNull + val batchId = ViewerRequestKeys.batchId(request).orNull - if (StringUtils.isBlank(collectionId)) { + if (StringUtils.isBlank(courseId)) { // delete all: fetch keys then delete each row val rows = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, new util.HashMap[String, AnyRef]() {{ put("userid", userId) }}, ctx) - rows.asScala.foreach(r => deleteEnrolment(userId, strOrNull(r.get("collectionid")), strOrNull(r.get("contextid")), ctx)) + rows.asScala.foreach(r => deleteEnrolment(userId, strOrNull(r.get("courseid")), strOrNull(r.get("batchid")), ctx)) } else { - deleteEnrolment(userId, collectionId, batchId, ctx) + deleteEnrolment(userId, courseId, batchId, ctx) } sender().tell(successResponse(), self) } - private def deleteEnrolment(userId: String, collectionId: String, batchId: String, ctx: RequestContext): Unit = { + private def deleteEnrolment(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = { val key = new util.HashMap[String, String]() key.put("userid", userId) - if (StringUtils.isNotBlank(collectionId)) key.put("collectionid", collectionId) - if (StringUtils.isNotBlank(batchId)) key.put("contextid", batchId) + if (StringUtils.isNotBlank(courseId)) key.put("courseid", courseId) + if (StringUtils.isNotBlank(batchId)) key.put("batchid", batchId) cassandraOperation.deleteRecord(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, key, ctx) } diff --git a/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java b/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java index 87c10496..739534e9 100644 --- a/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java +++ b/modules/viewer/service/app/controllers/viewer/ViewAggregateController.java @@ -43,12 +43,12 @@ public CompletionStage agg(Http.Request httpRequest) { private void validate(Request request) { String userId = (String) request.get(JsonKey.USER_ID); - Object collectionId = request.get("collectionId") != null ? request.get("collectionId") : request.get(JsonKey.COURSE_ID); + Object courseId = request.get(JsonKey.COURSE_ID); if (userId == null || userId.trim().isEmpty() - || collectionId == null || collectionId.toString().trim().isEmpty()) { + || courseId == null || courseId.toString().trim().isEmpty()) { throw new ProjectCommonException( ResponseCode.mandatoryParamsMissing.getErrorCode(), - "userId and collectionId (or courseId) are mandatory", + "userId and courseId are mandatory", ResponseCode.CLIENT_ERROR.getResponseCode()); } } diff --git a/modules/viewer/service/app/util/RequestValidator.java b/modules/viewer/service/app/util/RequestValidator.java index 5f359085..ec852723 100644 --- a/modules/viewer/service/app/util/RequestValidator.java +++ b/modules/viewer/service/app/util/RequestValidator.java @@ -70,8 +70,6 @@ public static void validateUpdateContent(Request contentRequestDto) { ERROR_CODE); } } - String courseId = map.containsKey(JsonKey.COURSE_ID) ? JsonKey.COURSE_ID : JsonKey.COLLECTION_ID; - map.put(JsonKey.COURSE_ID, map.get(courseId)); if (StringUtils.isBlank((String) map.get(JsonKey.COURSE_ID))) { throw new ProjectCommonException( ResponseCode.courseIdRequired, @@ -162,7 +160,7 @@ public static void validateUpdateContent(Request contentRequestDto) { } // Validation for enrolment sync if(CollectionUtils.isEmpty(list) && CollectionUtils.isEmpty(assessmentData)) { - contentRequestDto.getRequest().put(JsonKey.COURSE_ID, contentRequestDto.getOrDefault(JsonKey.COURSE_ID, contentRequestDto.getOrDefault(JsonKey.COLLECTION_ID, ""))); + contentRequestDto.getRequest().put(JsonKey.COURSE_ID, contentRequestDto.getOrDefault(JsonKey.COURSE_ID, "")); if (StringUtils.isBlank((String) contentRequestDto.getOrDefault(JsonKey.COURSE_ID, ""))) { throw new ProjectCommonException( ResponseCode.courseIdRequired, From 24a9bc7008838c17eb5c441cfbc403cd6f5d1d97 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Wed, 5 Aug 2026 15:06:29 +0530 Subject: [PATCH 18/30] fix: viewer enrolment rollup + lastcontentaccesstime 1) writeAllNodeEnrolments read row.get("courseid")/"batchid" but createResponse returns camelCase courseId/batchId, so nodeId was null and the root enrolment was never updated -> course completion never landed on user_enrolments. Read camelCase (lowercase fallback). 2) Stamp user_enrolments.lastcontentaccesstime/lastreadcontentid/lastreadcontentstatus on every view op (start/update/end/assess) so summary/list reflects real access. --- .../viewer/actor/ViewConsumptionActor.scala | 22 +++++++++++++++++++ .../viewer/actor/ViewerAggregatorActor.scala | 6 +++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala index 60d782ad..86433a32 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala @@ -39,6 +39,24 @@ class ViewConsumptionActor @Inject() ( private var cassandraOperation = ServiceFactory.getInstance private val consumptionDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_CONTENT_DB) private val CONSUMPTION_TABLE = "user_content_consumption" + private val enrolmentDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_COURSE_DB) + + /** + * Stamp the enrolment's last-content-access on every view op (mirrors standard content-consumption): + * user_enrolments.lastcontentaccesstime/lastreadcontentid/lastreadcontentstatus. Keyed by the ucc + * primary key (userid, courseid, batchid). This is what summary/list surfaces as access time. + */ + private def touchEnrolmentAccess(key: util.HashMap[String, AnyRef], status: Int, ctx: RequestContext): Unit = { + val selectMap = new util.HashMap[String, AnyRef]() {{ + put("userid", key.get("userid")); put("courseid", key.get("courseid")); put("batchid", key.get("batchid")) + }} + val updateMap = new util.HashMap[String, AnyRef]() {{ + put("lastcontentaccesstime", new java.util.Date()) + put("lastreadcontentid", key.get("contentid")) + put("lastreadcontentstatus", Integer.valueOf(status)) + }} + cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) + } // Assessment scoring reuses the assessment-aggregator services in-process (same math + persistence // the legacy AssessmentAggregatorActor uses). ContentService is only touched if metadata validation @@ -104,6 +122,7 @@ class ViewConsumptionActor @Inject() ( row.put("last_completed_time", ProjectUtil.getTimeStamp) row.put("last_updated_time", ProjectUtil.getTimeStamp) cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + touchEnrolmentAccess(key, 2, ctx) triggerAggregation(request, ctx) val out = new Response(); out.put(contentId, JsonKey.SUCCESS); sender().tell(out, self) @@ -176,6 +195,7 @@ class ViewConsumptionActor @Inject() ( cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) } // present -> already started, no-op + touchEnrolmentAccess(key, 1, ctx) sender().tell(successResponse(), self) } @@ -193,6 +213,7 @@ class ViewConsumptionActor @Inject() ( cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) } // absent -> update only if exists (ignore); already completed -> revisit ignored + touchEnrolmentAccess(key, math.max(1, if (existing != null) statusOf(existing) else 1), ctx) sender().tell(successResponse(), self) } @@ -205,6 +226,7 @@ class ViewConsumptionActor @Inject() ( row.put("last_completed_time", ProjectUtil.getTimeStamp) row.put("last_updated_time", ProjectUtil.getTimeStamp) cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + touchEnrolmentAccess(key, 2, ctx) // Async rollup: fire-and-forget tell to the aggregator; respond immediately (does not wait). triggerAggregation(request, ctx) sender().tell(successResponse(), self) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index e0f4c0b1..49f73d8b 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -295,8 +295,10 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) .asInstanceOf[util.List[util.Map[String, AnyRef]]] enrolRows.asScala.foreach { row => - val nodeId = Option(row.get("courseid")).map(_.toString).orNull - val nodeCtx = Option(row.get("batchid")).map(_.toString).orNull + // createResponse maps columns to camelCase field names (courseid->courseId, batchid->batchId via + // cassandratablecolumn.properties), so read camelCase (lowercase fallback for safety). + val nodeId = Option(row.get("courseId")).orElse(Option(row.get("courseid"))).map(_.toString).orNull + val nodeCtx = Option(row.get("batchId")).orElse(Option(row.get("batchid"))).map(_.toString).orNull // This LP only (root=batchId, child=batchId:childId); a standalone enrolment's batchid differs (§4). val expectedCtx = if (nodeId == rootId) batchId else batchId + ":" + nodeId nodeProgress.get(nodeId).filter(_ => nodeCtx == expectedCtx).foreach { case (completedCount, requiredLeaves) => From def10c9f90444d5b4b37a0dcacc4d05d13601f4e Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Wed, 5 Aug 2026 17:08:19 +0530 Subject: [PATCH 19/30] fix: summary CSV columns + drop dead lowercase fallback - summaryDownload CSV read lowercase keys (courseid/completionpercentage/completedon) but createResponse returns camelCase, so those columns were blank. Map csv headers to the camelCase result keys. - writeAllNodeEnrolments: createResponse rows are always camelCase, so drop the unnecessary lowercase orElse fallback and read courseId/batchId directly. --- .../sunbird/viewer/actor/ViewerAggregatorActor.scala | 8 ++++---- .../org/sunbird/viewer/actor/ViewerSummaryActor.scala | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index 49f73d8b..817846c8 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -295,10 +295,10 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]) .asInstanceOf[util.List[util.Map[String, AnyRef]]] enrolRows.asScala.foreach { row => - // createResponse maps columns to camelCase field names (courseid->courseId, batchid->batchId via - // cassandratablecolumn.properties), so read camelCase (lowercase fallback for safety). - val nodeId = Option(row.get("courseId")).orElse(Option(row.get("courseid"))).map(_.toString).orNull - val nodeCtx = Option(row.get("batchId")).orElse(Option(row.get("batchid"))).map(_.toString).orNull + // createResponse maps columns to camelCase field names (courseid->courseId, batchid->batchId + // via cassandratablecolumn.properties), so result rows are always camelCase. + val nodeId = Option(row.get("courseId")).map(_.toString).orNull + val nodeCtx = Option(row.get("batchId")).map(_.toString).orNull // This LP only (root=batchId, child=batchId:childId); a standalone enrolment's batchid differs (§4). val expectedCtx = if (nodeId == rootId) batchId else batchId + ":" + nodeId nodeProgress.get(nodeId).filter(_ => nodeCtx == expectedCtx).foreach { case (completedCount, requiredLeaves) => diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala index f3858c14..57d815ce 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala @@ -89,11 +89,15 @@ class ViewerSummaryActor extends BaseEnrolmentActor { sender().tell(response, self) } - private val csvCols = List("courseid", "batchid", "progress", "status", "completionpercentage", "completedon") + // (csv header, result-row key) — getRecords/createResponse returns camelCase field names + // (courseid->courseId, completionpercentage->completionPercentage, ...), so read those keys. + private val csvCols = List( + ("courseid", "courseId"), ("batchid", "batchId"), ("progress", "progress"), + ("status", "status"), ("completionpercentage", "completionPercentage"), ("completedon", "completedOn")) private def toCsv(rows: util.List[util.Map[String, AnyRef]]): String = { - val sb = new StringBuilder(csvCols.mkString(",")).append("\n") + val sb = new StringBuilder(csvCols.map(_._1).mkString(",")).append("\n") rows.asScala.foreach { r => - sb.append(csvCols.map(c => Option(r.get(c)).map(_.toString.replace(",", " ")).getOrElse("")).mkString(",")).append("\n") + sb.append(csvCols.map { case (_, key) => Option(r.get(key)).map(_.toString.replace(",", " ")).getOrElse("") }.mkString(",")).append("\n") } sb.toString } From dbc9ab5835606a3323f811aab58352bfdd4d3858 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Wed, 5 Aug 2026 17:54:05 +0530 Subject: [PATCH 20/30] feat: summary/download uploads CSV to cloud storage, returns url - summaryDownload (format=csv) writes CSV to a temp file and uploads via the generic CloudStorageUtil (StorageServiceFactory), returning result.url per the design. Provider/container/prefix all config-driven (sunbird_cloud_service_provider, sunbird_content_cloud_storage_container, viewer_summary_upload_path); no new dep. - CloudStorageUtil: pass cloud_storage_region to StorageConfig (AWS S3 outside us-east-1); no-op when blank so Azure/existing behaviour is unchanged. Add JsonKey.CLOUD_STORAGE_REGION + config default. - viewer/service pom: add azure/aws/gcp/oci profiles so the standalone viewer dist bundles the CSP runtime jar. --- .../main/java/org/sunbird/keys/JsonKey.java | 1 + .../org/sunbird/utils/CloudStorageUtil.java | 6 +++ .../resources/externalresource.properties | 4 ++ .../viewer/actor/ViewerSummaryActor.scala | 23 +++++++- modules/viewer/service/pom.xml | 52 +++++++++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) diff --git a/core/sunbird-platform-common/src/main/java/org/sunbird/keys/JsonKey.java b/core/sunbird-platform-common/src/main/java/org/sunbird/keys/JsonKey.java index 39b35767..885dfb2e 100644 --- a/core/sunbird-platform-common/src/main/java/org/sunbird/keys/JsonKey.java +++ b/core/sunbird-platform-common/src/main/java/org/sunbird/keys/JsonKey.java @@ -472,6 +472,7 @@ public final class JsonKey { public static final String CLOUD_FOLDER_CONTENT = "sunbird_cloud_content_folder"; public static final String CLOUD_STORE_BASE_PATH = "cloud_storage_base_url"; + public static final String CLOUD_STORAGE_REGION = "cloud_storage_region"; public static final String CLOUD_STORAGE_CNAME_URL= "cloud_storage_cname_url"; public static final String CLOUD_STORE_BASE_PATH_PLACEHOLDER = "cloud_store_base_path_placeholder"; public static final String TTL = "ttl"; diff --git a/core/sunbird-platform-common/src/main/java/org/sunbird/utils/CloudStorageUtil.java b/core/sunbird-platform-common/src/main/java/org/sunbird/utils/CloudStorageUtil.java index 7bc02b02..0f3a500f 100644 --- a/core/sunbird-platform-common/src/main/java/org/sunbird/utils/CloudStorageUtil.java +++ b/core/sunbird-platform-common/src/main/java/org/sunbird/utils/CloudStorageUtil.java @@ -175,6 +175,12 @@ private static IStorageService getStorageService( if (authType == StorageConfig.AuthType.ACCESS_KEY) { builder.storageSecret(storageSecret); } + // Region for providers that need it (e.g. AWS S3 outside us-east-1). No-op when unset: + // the provider SDK keeps its default (AWS -> us-east-1), so Azure/existing behaviour is unchanged. + String region = ProjectUtil.getConfigValue(JsonKey.CLOUD_STORAGE_REGION); + if (StringUtils.isNotBlank(region)) { + builder.region(region); + } StorageConfig storageConfig = builder.build(); IStorageService storageService = StorageServiceFactory.getStorageService(storageConfig); storageServiceMap.put(compositeKey, storageService); diff --git a/core/sunbird-platform-common/src/main/resources/externalresource.properties b/core/sunbird-platform-common/src/main/resources/externalresource.properties index d4d16ee5..975b627b 100644 --- a/core/sunbird-platform-common/src/main/resources/externalresource.properties +++ b/core/sunbird-platform-common/src/main/resources/externalresource.properties @@ -71,6 +71,8 @@ sunbird_api_request_lower_case_fields=source,externalId,userName,provider,loginI # Provide corresponding service provider container(azure,aws,gcloud) sunbird_content_cloud_storage_container=sunbird-content-dev sunbird_cloud_content_folder=content +# Object-key prefix (folder) for viewer summary CSV export; blank = container root +viewer_summary_upload_path=viewer-summary sunbird_time_zone=Asia/Kolkata sunbird_health_check_enable=true sunbird_sync_read_wait_time=1500 @@ -103,6 +105,8 @@ kafka_assessment_topic= sunbird_api_mgr_base_url=https://dev.sunbirded.org/api enrollment_list_size=1000 cloud_storage_base_url=https://sunbirddev.blob.core.windows.net +# Region for providers that need it (e.g. AWS S3 outside us-east-1); blank = provider default +cloud_storage_region= cloud_store_base_path_placeholder=CLOUD_BASE_PATH #Release-5.3.0 - LR-556 content_service_mock_enabled=false diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala index 57d815ce..e070542b 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala @@ -2,12 +2,14 @@ package org.sunbird.viewer.actor import org.apache.commons.collections4.CollectionUtils import org.apache.commons.lang3.StringUtils +import org.sunbird.common.ProjectUtil import org.sunbird.enrolments.BaseEnrolmentActor import org.sunbird.helper.ServiceFactory import org.sunbird.keys.JsonKey import org.sunbird.learner.util.Util import org.sunbird.request.{Request, RequestContext} import org.sunbird.response.Response +import org.sunbird.utils.CloudStorageUtil import java.util import scala.collection.JavaConverters._ @@ -84,11 +86,30 @@ class ViewerSummaryActor extends BaseEnrolmentActor { val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters, ctx) val response = new Response response.put("format", format) - if (format == "csv") response.put("content", toCsv(enrolments)) + if (format == "csv") response.put("url", uploadSummaryCsv(userId, toCsv(enrolments))) else response.put(JsonKey.RESPONSE, enrolments) sender().tell(response, self) } + /** + * Write the summary CSV to a temp file and upload it to cloud storage; return the object URL. + * Provider-agnostic via CloudStorageUtil (StorageServiceFactory). All config-driven, nothing hardcoded: + * sunbird_cloud_service_provider (provider), sunbird_content_cloud_storage_container (existing container), + * viewer_summary_upload_path (object-key prefix; blank = container root). Object = /_viewer_summary.csv + */ + private def uploadSummaryCsv(userId: String, csv: String): String = { + val storageType = ProjectUtil.getConfigValue("sunbird_cloud_service_provider") + val container = ProjectUtil.getConfigValue("sunbird_content_cloud_storage_container") + val prefix = Option(ProjectUtil.getConfigValue("viewer_summary_upload_path")).getOrElse("").trim.stripSuffix("/") + val objectKey = (if (StringUtils.isNotBlank(prefix)) prefix + "/" else "") + userId + "_viewer_summary.csv" + val tmp = java.io.File.createTempFile(userId + "_viewer_summary", ".csv") + try { + val w = new java.io.PrintWriter(tmp, "UTF-8") + try w.write(csv) finally w.close() + CloudStorageUtil.upload(storageType, container, objectKey, tmp.getAbsolutePath) + } finally tmp.delete() + } + // (csv header, result-row key) — getRecords/createResponse returns camelCase field names // (courseid->courseId, completionpercentage->completionPercentage, ...), so read those keys. private val csvCols = List( diff --git a/modules/viewer/service/pom.xml b/modules/viewer/service/pom.xml index 7a1a4314..9094a7b3 100644 --- a/modules/viewer/service/pom.xml +++ b/modules/viewer/service/pom.xml @@ -651,4 +651,56 @@ + + + + + azure + + true + + + + org.sunbird + cloud-storage-sdk-azure + 2.0.1 + runtime + + + + + aws + + + org.sunbird + cloud-storage-sdk-aws + 2.0.1 + runtime + + + + + gcp + + + org.sunbird + cloud-storage-sdk-gcp + 2.0.1 + runtime + + + + + oci + + + org.sunbird + cloud-storage-sdk-oci + 2.0.1 + runtime + + + + From 0a051794fbc015d94d5e8e1c56e9ba35141010f4 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Wed, 5 Aug 2026 17:56:52 +0530 Subject: [PATCH 21/30] fix: reverting git ignore --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index a0465425..d04e7644 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,3 @@ keys/ # Claude .claude/ -# viewer migration scripts kept local-only -modules/viewer/migrations/viewer.cql -modules/viewer/migrations/viewer-test-keyspace.cql From ed4d767a604c77d5591dfc58f1df4f16505a2fda Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Thu, 6 Aug 2026 10:42:29 +0530 Subject: [PATCH 22/30] fix(viewer): resolve code-review findings + LP rollup read consolidation Critical: - ViewConsumptionActor.touchEnrolmentAccess: read enrolment first, skip if absent (updateRecordV2 ifExists is a no-op -> would upsert phantom rows). - ViewerAggregatorActor: merge contentstatus into the existing map instead of replacing, so nodes outside the root-keyed read aren't clobbered. - ContentConsumptionActor.delegateAssessmentsToViewer: apply the legacy batch-validity guard (invalid/completed batches -> BATCH_NOT_EXISTS / NOT_A_ON_GOING_BATCH); only valid ongoing batches dispatch. Important: - ViewerSummaryActor.summaryDelete: read camelCase courseId/batchId. - ActorStartModule: bind aggregator with pekko.actor.viewer-dispatcher. - Delete dead RequestValidator.java and filters/LoggingFilter.java. - ViewController/ViewSummaryController: validate mandatory fields before dispatch (400 with offending field instead of NPE/500). - ContentConsumptionActor.updateConsumption: build finalContentList only in the legacy branch (dead work on the viewer-on path). Perf: - ViewerAggregatorActor.advanceLp: read enrolments once into a (courseId,batchId)->status snapshot; collapses the LP path's O(courses) status queries into one. Safe: runs after writeAllNodeEnrolments commits. --- .../enrolments/ContentConsumptionActor.scala | 97 +- .../viewer/actor/ViewConsumptionActor.scala | 5 + .../viewer/actor/ViewerAggregatorActor.scala | 48 +- .../viewer/actor/ViewerSummaryActor.scala | 8 +- .../controllers/viewer/ViewController.java | 29 + .../viewer/ViewSummaryController.java | 10 + .../service/app/filters/LoggingFilter.java | 42 - .../service/app/modules/ActorStartModule.java | 3 +- .../service/app/util/RequestValidator.java | 1018 ----------------- 9 files changed, 138 insertions(+), 1122 deletions(-) delete mode 100644 modules/viewer/service/app/filters/LoggingFilter.java delete mode 100644 modules/viewer/service/app/util/RequestValidator.java diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala index 85643b06..6f004730 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala @@ -88,31 +88,32 @@ class ContentConsumptionActor @Inject() ( val requestContext = request.getRequestContext val assessmentEvents = request.getRequest.getOrDefault(JsonKey.ASSESSMENT_EVENTS, new java.util.ArrayList[java.util.Map[String, AnyRef]]).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] val contentList = request.getRequest.getOrDefault(JsonKey.CONTENTS, new java.util.ArrayList[java.util.Map[String, AnyRef]]).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] - val finalContentList = if(CollectionUtils.isNotEmpty(assessmentEvents)) { - logger.info(requestContext, "Assessment Consumption events exist: " + assessmentEvents.size()) - val assessmentConsumptions = assessmentEvents.map(e => { - InternalContentConsumption(e.get("courseId").asInstanceOf[String], e.get("batchId").asInstanceOf[String], e.get("contentId").asInstanceOf[String]) - }).filter(cc => cc.validConsumption()).map(cc => { - var consumption: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]() - consumption.put("courseId", cc.courseId) - consumption.put("batchId", cc.batchId) - consumption.put("contentId", cc.contentId) - consumption.put("status", 2.asInstanceOf[AnyRef]) - consumption - }) - if (CollectionUtils.isNotEmpty(contentList)) (contentList ++ assessmentConsumptions).asJava else assessmentConsumptions.asJava - } else contentList - logger.info(requestContext, "Final content-consumption data: " + finalContentList) - // Update consumption first and then push the assessment events if there are any. This will help us handling failures of max attempts (for assessment content). // viewer.enabled -> the Viewer Service OWNS both consumption AND assessment processing: // regular contents -> /v1/view/start|end (ucc write + recursive rollup) // assessment events -> /v1/assessment/submit (score + status=2 + same rollup) - // Assessments are therefore NOT merged into the content list here (submit marks completion - // itself, so no double rollup) and legacy processContents/processAssessments are skipped. - // API contract (per-key SUCCESS map) is preserved. + // Assessments are NOT merged into the content list here (submit marks completion itself, no + // double rollup) and legacy processContents/processAssessments are skipped. API contract preserved. val contentConsumptionResponse = if (isViewerEnabled) delegateContentsToViewer(contentList, request, requestBy, requestedFor) - else processContents(finalContentList, requestContext, requestBy, requestedFor) + else { + // legacy path only: merge assessment events (as completed contents) into the content list. + val finalContentList = if(CollectionUtils.isNotEmpty(assessmentEvents)) { + logger.info(requestContext, "Assessment Consumption events exist: " + assessmentEvents.size()) + val assessmentConsumptions = assessmentEvents.map(e => { + InternalContentConsumption(e.get("courseId").asInstanceOf[String], e.get("batchId").asInstanceOf[String], e.get("contentId").asInstanceOf[String]) + }).filter(cc => cc.validConsumption()).map(cc => { + val consumption: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]() + consumption.put("courseId", cc.courseId) + consumption.put("batchId", cc.batchId) + consumption.put("contentId", cc.contentId) + consumption.put("status", 2.asInstanceOf[AnyRef]) + consumption + }) + if (CollectionUtils.isNotEmpty(contentList)) (contentList ++ assessmentConsumptions).asJava else assessmentConsumptions.asJava + } else contentList + logger.info(requestContext, "Final content-consumption data: " + finalContentList) + processContents(finalContentList, requestContext, requestBy, requestedFor) + } val assessmentResponse = if (isViewerEnabled) delegateAssessmentsToViewer(assessmentEvents, request, requestBy, requestedFor) else processAssessments(assessmentEvents, requestContext, requestBy, requestedFor) @@ -528,8 +529,10 @@ class ContentConsumptionActor @Inject() ( /** * Backward-compat adapter for content/state/update (assessment events): dispatch each to the viewer - * assessment/submit (score + status=2 + rollup). courseId/batchId map to collectionId/contextId. - * Response: batchId -> SUCCESS/FAILED. + * assessment/submit (score + status=2 + rollup). Applies the SAME batch-validity guard as legacy + * processAssessments: unknown batch -> BATCH_NOT_EXISTS, completed batch (status != 1) -> + * NOT_A_ON_GOING_BATCH; only ongoing batches are dispatched. Response: batchId -> SUCCESS/FAILED + * (+ the two error keys when applicable). */ private def delegateAssessmentsToViewer(assessmentEvents: java.util.List[java.util.Map[String, AnyRef]], originalRequest: Request, @@ -538,26 +541,40 @@ class ContentConsumptionActor @Inject() ( val ctx = originalRequest.getRequestContext val userId = if (StringUtils.isNotBlank(requestedFor)) requestedFor else requestedBy val token = originalRequest.getContext.get(JsonKey.X_AUTH_TOKEN).asInstanceOf[String] + val byBatch: Map[String, List[java.util.Map[String, AnyRef]]] = assessmentEvents.asScala + .filter(e => StringUtils.isNotBlank(e.getOrDefault(JsonKey.BATCH_ID, "").asInstanceOf[String])).toList + .groupBy(_.get(JsonKey.BATCH_ID).asInstanceOf[String]) + val batchIds = byBatch.keySet.toList.asJava + val batches: Map[String, List[java.util.Map[String, AnyRef]]] = + getBatches(ctx, new java.util.ArrayList[String](batchIds), null).toList.groupBy(_.get(JsonKey.BATCH_ID).asInstanceOf[String]) + val invalidBatchIds = byBatch.keySet.diff(batches.keySet).toList.asJava + batches.values.foreach(bl => bl.foreach(b => CourseBatchUtil.enrichBatchStatusFromDates(b))) + val completedBatchIds = batches.filter(b => 1 != b._2.head.get(JsonKey.STATUS).asInstanceOf[Integer]).keys.toList.asJava val responseMessage = new java.util.HashMap[String, AnyRef]() - assessmentEvents.asScala.foreach(a => { - val batchId = a.getOrDefault(JsonKey.BATCH_ID, "").asInstanceOf[String] - try { - val courseId = a.get(JsonKey.COURSE_ID).asInstanceOf[String] - val events = a.getOrDefault(JsonKey.ASSESSMENT_EVENTS_KEY, new java.util.ArrayList[java.util.Map[String, AnyRef]]()) - val body = new java.util.HashMap[String, AnyRef]() {{ - put("contentId", a.get(JsonKey.CONTENT_ID)) - put("courseId", courseId) - put("batchId", batchId) - put(JsonKey.USER_ID, userId) - put(JsonKey.ASSESSMENT_EVENTS, events) - }} - responseMessage.put(batchId, if (viewerWrite("view-consumption-actor", "/v1/assessment/submit", "viewAssess", body, token, ctx)) JsonKey.SUCCESS else "FAILED") - } catch { - case ex: Exception => - logger.error(ctx, s"delegateAssessmentsToViewer failed for batchId=$batchId: ${ex.getMessage}", ex) - responseMessage.put(batchId, "FAILED") + byBatch.foreach { case (batchId, events) => + if (!invalidBatchIds.contains(batchId) && !completedBatchIds.contains(batchId)) { + events.foreach(a => { + try { + val courseId = a.get(JsonKey.COURSE_ID).asInstanceOf[String] + val evs = a.getOrDefault(JsonKey.ASSESSMENT_EVENTS_KEY, new java.util.ArrayList[java.util.Map[String, AnyRef]]()) + val body = new java.util.HashMap[String, AnyRef]() {{ + put("contentId", a.get(JsonKey.CONTENT_ID)) + put("courseId", courseId) + put("batchId", batchId) + put(JsonKey.USER_ID, userId) + put(JsonKey.ASSESSMENT_EVENTS, evs) + }} + responseMessage.put(batchId, if (viewerWrite("view-consumption-actor", "/v1/assessment/submit", "viewAssess", body, token, ctx)) JsonKey.SUCCESS else "FAILED") + } catch { + case ex: Exception => + logger.error(ctx, s"delegateAssessmentsToViewer failed for batchId=$batchId: ${ex.getMessage}", ex) + responseMessage.put(batchId, "FAILED") + } + }) } - }) + } + if (CollectionUtils.isNotEmpty(completedBatchIds)) responseMessage.put("NOT_A_ON_GOING_BATCH", completedBatchIds) + if (CollectionUtils.isNotEmpty(invalidBatchIds)) responseMessage.put("BATCH_NOT_EXISTS", invalidBatchIds) val response = new Response(); response.putAll(responseMessage); Option(response) } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala index 86433a32..0176b34e 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala @@ -50,6 +50,11 @@ class ViewConsumptionActor @Inject() ( val selectMap = new util.HashMap[String, AnyRef]() {{ put("userid", key.get("userid")); put("courseid", key.get("courseid")); put("batchid", key.get("batchid")) }} + // Only stamp an EXISTING enrolment. updateRecordV2's ifExists is a no-op (plain UPDATE upserts in + // Cassandra), so without this guard a no-context/unenrolled view would fabricate a phantom enrolment row. + val existing = cassandraOperation.getRecordByIdentifier(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (existing.isEmpty) return val updateMap = new util.HashMap[String, AnyRef]() {{ put("lastcontentaccesstime", new java.util.Date()) put("lastreadcontentid", key.get("contentid")) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index 817846c8..72bb9a71 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -139,14 +139,20 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { private def advanceLp(userId: String, rootId: String, batchId: String, trackable: List[String], ctx: RequestContext): Unit = { - ensureOptionalityComputed(userId, rootId, batchId, trackable, ctx) + // One read of this user's enrolments, reused for every completion/enrolment check below. Safe because + // advanceLp runs AFTER writeAllNodeEnrolments has committed this pass's statuses, so the snapshot is + // current; collapses the LP's former O(courses) single-row status reads into a single query. + val status = enrolStatusSnapshot(userId, ctx) + val childBatchOf = (c: String) => batchId + ":" + c + val courseComplete = (c: String) => status.get((c, childBatchOf(c))).contains(2) + + ensureOptionalityComputed(userId, rootId, batchId, trackable, courseComplete, ctx) val optional = readOptionalNodes(userId, rootId, batchId, ctx).toSet val ancestorsOf = (n: String) => hierarchyRelationsUtil.getAncestors(rootId, n, ctx) val levels = ProgressionPolicy.orderedLevels(trackable, ancestorsOf, rootId) // Level complete = all its required (non-optional) courses complete (empty required set = complete, §5). // Derived from persisted enrolment status only, so it's recompute-safe (force-sync repairs identically). - def courseComplete(c: String): Boolean = enrolStatus(userId, c, batchId + ":" + c, ctx).contains(2) def levelComplete(level: String): Boolean = ProgressionPolicy.coursesOfLevel(level, trackable, ancestorsOf, rootId).filterNot(optional.contains).forall(courseComplete) @@ -155,8 +161,8 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { val courses = ProgressionPolicy.coursesOfLevel(level, trackable, ancestorsOf, rootId) val nextRequired = courses.filterNot(optional.contains).find(c => !courseComplete(c)) (courses.filter(optional.contains) ++ nextRequired.toList).foreach { c => - val childBatch = batchId + ":" + c - if (!isEnrolled(userId, c, childBatch, ctx)) internalEnrol(userId, c, childBatch, ctx) + val childBatch = childBatchOf(c) + if (!status.contains((c, childBatch))) internalEnrol(userId, c, childBatch, ctx) } } @@ -164,14 +170,15 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { if (levels.nonEmpty && levels.forall(levelComplete)) creditSkills(userId, rootId, trackable, ctx) } - private def ensureOptionalityComputed(userId: String, rootId: String, batchId: String, trackable: List[String], ctx: RequestContext): Unit = { + private def ensureOptionalityComputed(userId: String, rootId: String, batchId: String, trackable: List[String], + courseComplete: String => Boolean, ctx: RequestContext): Unit = { // Compute once (§Step 4); optionalityComputed remembers empty results without a DB column. if (optionalityComputed(userId, rootId, batchId, ctx)) return val policy = policyOf(rootId, ctx) if (policy.equalsIgnoreCase("Strict") || trackable.isEmpty) { writeOptionalNodes(userId, rootId, batchId, Set.empty, ctx); return } val diagnostic = trackable.head val hasDiagnostic = isAssessmentCourse(diagnostic, ctx) - if (hasDiagnostic && !isComplete(userId, diagnostic, batchId, ctx)) return // wait for the diagnostic + if (hasDiagnostic && !courseComplete(diagnostic)) return // wait for the diagnostic val prior = if (policy.equalsIgnoreCase("PriorLearning")) readUserSkills(userId, ctx) else Set.empty[String] val fromDiag = if (hasDiagnostic) skillsFromAssessment(userId, rootId, diagnostic, ctx) else Set.empty[String] val achieved = prior ++ fromDiag @@ -192,20 +199,24 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // VERIFY-ON-DEPLOY: derive from best-attempt assessment_aggregator scores × se_skills tags (skill achieved = all its questions correct). private def skillsFromAssessment(userId: String, rootId: String, courseId: String, ctx: RequestContext): Set[String] = Set.empty - private def isComplete(userId: String, courseId: String, rootBatchId: String, ctx: RequestContext): Boolean = - enrolStatus(userId, courseId, rootBatchId + ":" + courseId, ctx).contains(2) - - private def enrolStatus(userId: String, courseId: String, batchId: String, ctx: RequestContext): Option[Int] = { - val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", courseId); put("batchid", batchId) }} + /** + * All of this user's enrolments as (courseId, batchId) -> status, in one read. Replaces the LP's former + * per-course status queries. Result rows are camelCase (createResponse); rows without a status map to 0 + * (enrol always sets one) so presence-of-key still answers "is enrolled". + */ + private def enrolStatusSnapshot(userId: String, ctx: RequestContext): Map[(String, String), Int] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} val rows = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] - if (CollectionUtils.isNotEmpty(rows)) Option(rows.get(0).get("status")).map(_.asInstanceOf[Number].intValue()) else None + rows.asScala.flatMap { r => + for { + c <- Option(r.get("courseId")).map(_.toString) + b <- Option(r.get("batchId")).map(_.toString) + } yield (c, b) -> Option(r.get("status")).map(_.asInstanceOf[Number].intValue()).getOrElse(0) + }.toMap } - private def isEnrolled(userId: String, courseId: String, batchId: String, ctx: RequestContext): Boolean = - enrolStatus(userId, courseId, batchId, ctx).isDefined - /** * Internal (system-driven) enrol via the ProgressionEnroller gateway — the FULL enrol op (`doEnrol`), * NOT a bare DAO write: it reuses CourseEnrolmentActor's verified write path (DB + cache + telemetry). @@ -307,6 +318,11 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { val pct = activityAggUtil.getCompletionPercentage(completedCount, required) val currentStatus = Option(row.get("status")).map(_.asInstanceOf[Number].intValue()).getOrElse(0) val nodeContentStatus = requiredLeaves.flatMap(l => contentStatusMap.get(l).map(cs => l -> Integer.valueOf(cs.status))).toMap + // contentstatus is a full-column replace in updateRecordV2 — merge into the row's existing map so + // leaves not in this (root-keyed) read aren't clobbered. + val mergedContentStatus = new util.HashMap[String, AnyRef]() + Option(row.get("contentStatus")).foreach(m => mergedContentStatus.putAll(m.asInstanceOf[util.Map[String, AnyRef]])) + nodeContentStatus.foreach { case (k, v) => mergedContentStatus.put(k, v) } val selectMap = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", nodeId); put("batchid", nodeCtx) }} @@ -314,7 +330,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { put("progress", Integer.valueOf(completedCount)) put("status", Integer.valueOf(status)) put("completionpercentage", Integer.valueOf(pct)) - put("contentstatus", nodeContentStatus.asJava) + put("contentstatus", mergedContentStatus) if (status == 2 && currentStatus != 2) put("completedon", new java.util.Date()) }} cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala index e070542b..16942a04 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala @@ -72,10 +72,8 @@ class ViewerSummaryActor extends BaseEnrolmentActor { } /** - * Exhaust download of a user's enrolment summaries. format=json (default) returns the rows; - * format=csv returns a CSV string under "content". ponytail: inline export (no cloud upload / signed - * URL) — fine for per-user summaries; switch to cloud-storage-sdk + a returned URL if exhaust grows - * large or needs a stored artifact. + * Download a user's enrolment summaries. format=json (default) returns the rows under "response"; + * format=csv uploads the CSV to cloud storage (generic CloudStorageUtil) and returns its "url". */ private def summaryDownload(request: Request): Unit = { val ctx = request.getRequestContext @@ -135,7 +133,7 @@ class ViewerSummaryActor extends BaseEnrolmentActor { // delete all: fetch keys then delete each row val rows = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, new util.HashMap[String, AnyRef]() {{ put("userid", userId) }}, ctx) - rows.asScala.foreach(r => deleteEnrolment(userId, strOrNull(r.get("courseid")), strOrNull(r.get("batchid")), ctx)) + rows.asScala.foreach(r => deleteEnrolment(userId, strOrNull(r.get("courseId")), strOrNull(r.get("batchId")), ctx)) } else { deleteEnrolment(userId, courseId, batchId, ctx) } diff --git a/modules/viewer/service/app/controllers/viewer/ViewController.java b/modules/viewer/service/app/controllers/viewer/ViewController.java index c47b65f4..72e0d159 100644 --- a/modules/viewer/service/app/controllers/viewer/ViewController.java +++ b/modules/viewer/service/app/controllers/viewer/ViewController.java @@ -1,7 +1,10 @@ package controllers.viewer; import controllers.BaseController; +import org.apache.commons.lang3.StringUtils; import org.apache.pekko.actor.ActorRef; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.message.ResponseCode; import org.sunbird.request.Request; import play.mvc.Http; import play.mvc.Result; @@ -67,9 +70,35 @@ public Result preflight(String all) { private CompletionStage dispatch(String operation, Http.Request httpRequest) { try { Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + validate(operation, request); return actorResponseHandler(viewConsumptionActor, request, timeout, null, httpRequest); } catch (Exception e) { return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); } } + + // Reject requests missing keys the actor unconditionally dereferences, so callers get a 400 with the + // offending field instead of an opaque 500/NPE. Contract is courseId/batchId/contentId (+ userId). + private void validate(String operation, Request request) { + switch (operation) { + case "viewStart": case "viewUpdate": case "viewEnd": case "viewAssess": case "assessmentRead": + requireNonBlank(request, "userId", "courseId", "batchId", "contentId"); + break; + case "viewRead": + requireNonBlank(request, "userId", "courseId", "batchId"); + break; + default: // no mandatory fields + } + } + + private void requireNonBlank(Request request, String... keys) { + for (String key : keys) { + if (StringUtils.isBlank((String) request.getRequest().get(key))) { + throw new ProjectCommonException( + ResponseCode.mandatoryParameterMissing.getErrorCode(), + ResponseCode.mandatoryParameterMissing.getErrorMessage() + " " + key, + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } + } } diff --git a/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java b/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java index 2d8dc27e..1701f4e3 100644 --- a/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java +++ b/modules/viewer/service/app/controllers/viewer/ViewSummaryController.java @@ -1,7 +1,10 @@ package controllers.viewer; import controllers.BaseController; +import org.apache.commons.lang3.StringUtils; import org.apache.pekko.actor.ActorRef; +import org.sunbird.exception.ProjectCommonException; +import org.sunbird.message.ResponseCode; import org.sunbird.request.Request; import play.mvc.Http; import play.mvc.Result; @@ -67,6 +70,13 @@ public CompletionStage summaryDelete(String userId, Http.Request httpReq private CompletionStage dispatchBody(String operation, Http.Request httpRequest) { try { Request request = createAndInitRequest(operation, httpRequest.body().asJson(), httpRequest); + // summaryRead identifies the enrolment by userId (from the body) — reject if absent. + if (StringUtils.isBlank((String) request.getRequest().get("userId"))) { + throw new ProjectCommonException( + ResponseCode.mandatoryParameterMissing.getErrorCode(), + ResponseCode.mandatoryParameterMissing.getErrorMessage() + " userId", + ResponseCode.CLIENT_ERROR.getResponseCode()); + } return actorResponseHandler(viewerSummaryActor, request, timeout, null, httpRequest); } catch (Exception e) { return CompletableFuture.completedFuture(createCommonExceptionResponse(e, httpRequest)); diff --git a/modules/viewer/service/app/filters/LoggingFilter.java b/modules/viewer/service/app/filters/LoggingFilter.java deleted file mode 100644 index f45b9c88..00000000 --- a/modules/viewer/service/app/filters/LoggingFilter.java +++ /dev/null @@ -1,42 +0,0 @@ -package filters; - -import org.apache.pekko.stream.Materializer; -import play.Logger; -import play.mvc.Filter; -import play.mvc.Http; -import play.mvc.Result; - -import javax.inject.Inject; -import java.util.concurrent.CompletionStage; -import java.util.function.Function; - -public class LoggingFilter extends Filter { - - @Inject - public LoggingFilter(Materializer mat) { - super(mat); - } - - @Override - public CompletionStage apply( - Function> nextFilter, - Http.RequestHeader requestHeader) { - long startTime = System.currentTimeMillis(); - return nextFilter - .apply(requestHeader) - .thenApply( - result -> { - long endTime = System.currentTimeMillis(); - long requestTime = endTime - startTime; - - Logger.info( - "{} {} took {}ms and returned {}", - requestHeader.method(), - requestHeader.uri(), - requestTime, - result.status()); - - return result.withHeader("Request-Time", "" + requestTime); - }); - } -} \ No newline at end of file diff --git a/modules/viewer/service/app/modules/ActorStartModule.java b/modules/viewer/service/app/modules/ActorStartModule.java index 0446791a..b0bd2624 100644 --- a/modules/viewer/service/app/modules/ActorStartModule.java +++ b/modules/viewer/service/app/modules/ActorStartModule.java @@ -32,7 +32,8 @@ protected void configure() { bindActor( actor.getActorClass(), actor.getActorName(), - props -> props.withRouter(new ConsistentHashingPool(8).withHashMapper(userIdHashMapper))); + props -> props.withRouter(new ConsistentHashingPool(8).withHashMapper(userIdHashMapper)) + .withDispatcher("pekko.actor.viewer-dispatcher")); } else { bindActor(actor.getActorClass(), actor.getActorName(), props -> props.withRouter(config)); } diff --git a/modules/viewer/service/app/util/RequestValidator.java b/modules/viewer/service/app/util/RequestValidator.java deleted file mode 100644 index ec852723..00000000 --- a/modules/viewer/service/app/util/RequestValidator.java +++ /dev/null @@ -1,1018 +0,0 @@ -package util; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.StringUtils; -import org.sunbird.exception.ProjectCommonException; -import org.sunbird.telemetry.dto.*; -import org.sunbird.common.ProjectUtil.ProgressStatus; -import org.sunbird.common.ProjectUtil.Source; -import org.sunbird.keys.JsonKey; -import org.sunbird.common.ProjectUtil; -import org.sunbird.common.PropertiesCache; -import org.sunbird.utils.StringFormatter; -import org.sunbird.request.Request; -import org.sunbird.response.ResponseCode; -import org.sunbird.response.ResponseMessage; -import org.sunbird.logging.LoggerUtil; - -import java.text.MessageFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Map; - -/** - * This call will do validation for all incoming request data. - * - * @author Manzarul - */ -public final class RequestValidator { - private static final int ERROR_CODE = ResponseCode.CLIENT_ERROR.getResponseCode(); - public static LoggerUtil logger = new LoggerUtil(RequestValidator.class); - - private RequestValidator() {} - - /** - * This method will do content state request data validation. if all mandatory data is coming then - * it won't do any thing if any mandatory data is missing then it will throw exception. - * - * @param contentRequestDto Request - */ - @SuppressWarnings("unchecked") - public static void validateUpdateContent(Request contentRequestDto) { - List> list = - (List>) (contentRequestDto.getRequest().get(JsonKey.CONTENTS)); - if(CollectionUtils.isNotEmpty(list)) { - for (Map map : list) { - if (null != map.get(JsonKey.LAST_UPDATED_TIME)) { - boolean bool = - ProjectUtil.isDateValidFormat( - "yyyy-MM-dd HH:mm:ss:SSSZ", (String) map.get(JsonKey.LAST_UPDATED_TIME)); - if (!bool) { - throw new ProjectCommonException( - ResponseCode.dateFormatError, - ResponseCode.dateFormatError.getErrorMessage(), - ERROR_CODE); - } - } - if (null != map.get(JsonKey.LAST_COMPLETED_TIME)) { - boolean bool = - ProjectUtil.isDateValidFormat( - "yyyy-MM-dd HH:mm:ss:SSSZ", (String) map.get(JsonKey.LAST_COMPLETED_TIME)); - if (!bool) { - throw new ProjectCommonException( - ResponseCode.dateFormatError, - ResponseCode.dateFormatError.getErrorMessage(), - ERROR_CODE); - } - } - if (StringUtils.isBlank((String) map.get(JsonKey.COURSE_ID))) { - throw new ProjectCommonException( - ResponseCode.courseIdRequired, - ResponseCode.courseIdRequiredError.getErrorMessage(), - ERROR_CODE); - } - if (map.containsKey(JsonKey.CONTENT_ID)) { - - if (null == map.get(JsonKey.CONTENT_ID)) { - throw new ProjectCommonException( - ResponseCode.contentIdRequired, - ResponseCode.contentIdRequiredError.getErrorMessage(), - ERROR_CODE); - } - if (ProjectUtil.isNull(map.get(JsonKey.STATUS))) { - throw new ProjectCommonException( - ResponseCode.contentStatusRequired, - ResponseCode.contentStatusRequired.getErrorMessage(), - ERROR_CODE); - } - - } else { - throw new ProjectCommonException( - ResponseCode.contentIdRequired, - ResponseCode.contentIdRequiredError.getErrorMessage(), - ERROR_CODE); - } - } - } - List> assessmentData = - (List>) contentRequestDto.getRequest().get(JsonKey.ASSESSMENT_EVENTS); - if (CollectionUtils.isNotEmpty(assessmentData)) { - for (Map map : assessmentData) { - if (!map.containsKey(JsonKey.ASSESSMENT_TS)) { - throw new ProjectCommonException( - ResponseCode.assessmentAttemptDateRequired, - ResponseCode.assessmentAttemptDateRequired.getErrorMessage(), - ERROR_CODE); - } - - if (!map.containsKey(JsonKey.COURSE_ID) - || StringUtils.isBlank((String) map.get(JsonKey.COURSE_ID))) { - throw new ProjectCommonException( - ResponseCode.courseIdRequired, - ResponseCode.courseIdRequiredError.getErrorMessage(), - ERROR_CODE); - } - - if (!map.containsKey(JsonKey.CONTENT_ID) - || StringUtils.isBlank((String) map.get(JsonKey.CONTENT_ID))) { - throw new ProjectCommonException( - ResponseCode.contentIdRequired, - ResponseCode.contentIdRequiredError.getErrorMessage(), - ERROR_CODE); - } - - if (!map.containsKey(JsonKey.BATCH_ID) - || StringUtils.isBlank((String) map.get(JsonKey.BATCH_ID))) { - throw new ProjectCommonException( - ResponseCode.courseBatchIdRequired, - ResponseCode.courseBatchIdRequired.getErrorMessage(), - ERROR_CODE); - } - - if (!map.containsKey(JsonKey.USER_ID) - || StringUtils.isBlank((String) map.get(JsonKey.USER_ID))) { - throw new ProjectCommonException( - ResponseCode.userIdRequired, - ResponseCode.userIdRequired.getErrorMessage(), - ERROR_CODE); - } - - if (!map.containsKey(JsonKey.ATTEMPT_ID) - || StringUtils.isBlank((String) map.get(JsonKey.ATTEMPT_ID))) { - throw new ProjectCommonException( - ResponseCode.attemptIdRequired, - ResponseCode.attemptIdRequired.getErrorMessage(), - ERROR_CODE); - } - - if (!map.containsKey(JsonKey.EVENTS)) { - throw new ProjectCommonException( - ResponseCode.eventsRequired, - ResponseCode.eventsRequired.getErrorMessage(), - ERROR_CODE); - } - } - } - // Validation for enrolment sync - if(CollectionUtils.isEmpty(list) && CollectionUtils.isEmpty(assessmentData)) { - contentRequestDto.getRequest().put(JsonKey.COURSE_ID, contentRequestDto.getOrDefault(JsonKey.COURSE_ID, "")); - if (StringUtils.isBlank((String) contentRequestDto.getOrDefault(JsonKey.COURSE_ID, ""))) { - throw new ProjectCommonException( - ResponseCode.courseIdRequired, - ResponseCode.courseIdRequiredError.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank((String) contentRequestDto.getOrDefault(JsonKey.BATCH_ID, ""))) { - throw new ProjectCommonException( - ResponseCode.courseBatchIdRequired, - ResponseCode.courseBatchIdRequired.getErrorMessage(), - ERROR_CODE); - } - - if (StringUtils.isBlank((String) contentRequestDto.getOrDefault(JsonKey.USER_ID, ""))) { - throw new ProjectCommonException( - ResponseCode.userIdRequired, - ResponseCode.userIdRequired.getErrorMessage(), - ERROR_CODE); - } - } - } - - /** - * This method will validate get page data api. - * - * @param request Request - */ - public static void validateGetPageData(Request request) { - if (request == null || (StringUtils.isBlank((String) request.get(JsonKey.SOURCE)))) { - throw new ProjectCommonException( - ResponseCode.sourceRequired, - ResponseCode.sourceRequired.getErrorMessage(), - ERROR_CODE); - } - if (!validPageSourceType((String) request.get(JsonKey.SOURCE))) { - throw new ProjectCommonException( - ResponseCode.invalidPageSource, - ResponseCode.invalidPageSource.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank((String) request.get(JsonKey.PAGE_NAME))) { - throw new ProjectCommonException( - ResponseCode.pageNameRequired, - ResponseCode.pageNameRequired.getErrorMessage(), - ERROR_CODE); - } - } - - private static boolean validPageSourceType(String source) { - - Boolean isValidSource = false; - for (Source src : Source.values()) { - if (src.getValue().equalsIgnoreCase(source)) { - isValidSource = true; - break; - } - } - return isValidSource; - } - - /** - * This method will validate add course request data. - * - * @param courseRequest Request - */ - public static void validateAddBatchCourse(Request courseRequest) { - - if (courseRequest.getRequest().get(JsonKey.BATCH_ID) == null) { - throw new ProjectCommonException( - ResponseCode.courseBatchIdRequired, - ResponseCode.courseBatchIdRequired.getErrorMessage(), - ERROR_CODE); - } - if (courseRequest.getRequest().get(JsonKey.USER_IDs) == null) { - throw new ProjectCommonException( - ResponseCode.userIdRequired, - ResponseCode.userIdRequired.getErrorMessage(), - ERROR_CODE); - } - } - - /** - * This method will validate add course request data. - * - * @param courseRequest Request - */ - public static void validateGetBatchCourse(Request courseRequest) { - - if (courseRequest.getRequest().get(JsonKey.BATCH_ID) == null) { - throw new ProjectCommonException( - ResponseCode.courseBatchIdRequired, - ResponseCode.courseBatchIdRequired.getErrorMessage(), - ERROR_CODE); - } - } - - /** - * This method will validate update course request data. - * - * @param request Request - */ - public static void validateUpdateCourse(Request request) { - - if (request.getRequest().get(JsonKey.COURSE_ID) == null) { - throw new ProjectCommonException( - ResponseCode.courseIdRequired, - ResponseCode.courseIdRequired.getErrorMessage(), - ERROR_CODE); - } - } - - /** - * This method will validate published course request data. - * - * @param request Request - */ - public static void validatePublishCourse(Request request) { - if (request.getRequest().get(JsonKey.COURSE_ID) == null) { - throw new ProjectCommonException( - ResponseCode.courseIdRequiredError, - ResponseCode.courseIdRequiredError.getErrorMessage(), - ERROR_CODE); - } - } - - /** - * This method will validate Delete course request data. - * - * @param request Request - */ - public static void validateDeleteCourse(Request request) { - if (request.getRequest().get(JsonKey.COURSE_ID) == null) { - throw new ProjectCommonException( - ResponseCode.courseIdRequiredError, - ResponseCode.courseIdRequiredError.getErrorMessage(), - ERROR_CODE); - } - } - - /* - * This method will validate create section data - * - * @param userRequest Request - */ - public static void validateCreateSection(Request request) { - if (StringUtils.isBlank( - (String) - (request.getRequest().get(JsonKey.SECTION_NAME) != null - ? request.getRequest().get(JsonKey.SECTION_NAME) - : ""))) { - throw new ProjectCommonException( - ResponseCode.sectionNameRequired, - ResponseCode.sectionNameRequired.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank( - (String) - (request.getRequest().get(JsonKey.SECTION_DATA_TYPE) != null - ? request.getRequest().get(JsonKey.SECTION_DATA_TYPE) - : ""))) { - throw new ProjectCommonException( - ResponseCode.sectionDataTypeRequired, - ResponseCode.sectionDataTypeRequired.getErrorMessage(), - ERROR_CODE); - } - } - - /** - * This method will validate update section request data - * - * @param request Request - */ - public static void validateUpdateSection(Request request) { - if (request.getRequest().containsKey(JsonKey.SECTION_NAME) - && StringUtils.isBlank( - (String) - (request.getRequest().get(JsonKey.SECTION_NAME) != null - ? request.getRequest().get(JsonKey.SECTION_NAME) - : ""))) { - throw new ProjectCommonException( - ResponseCode.sectionNameRequired, - ResponseCode.sectionNameRequired.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank( - (String) - (request.getRequest().get(JsonKey.ID) != null - ? request.getRequest().get(JsonKey.ID) - : ""))) { - throw new ProjectCommonException( - ResponseCode.sectionIdRequired, - ResponseCode.sectionIdRequired.getErrorMessage(), - ERROR_CODE); - } - if (request.getRequest().containsKey(JsonKey.SECTION_DATA_TYPE) - && StringUtils.isBlank( - (String) - (request.getRequest().get(JsonKey.SECTION_DATA_TYPE) != null - ? request.getRequest().get(JsonKey.SECTION_DATA_TYPE) - : ""))) { - throw new ProjectCommonException( - ResponseCode.sectionDataTypeRequired, - ResponseCode.sectionDataTypeRequired.getErrorMessage(), - ERROR_CODE); - } - } - - /** - * This method will validate create page data - * - * @param request Request - */ - public static void validateCreatePage(Request request) { - if (StringUtils.isEmpty( - (String) - (request.getRequest().get(JsonKey.PAGE_NAME) != null - ? request.getRequest().get(JsonKey.PAGE_NAME) - : ""))) { - throw new ProjectCommonException( - ResponseCode.pageNameRequired, - ResponseCode.pageNameRequired.getErrorMessage(), - ERROR_CODE); - } - } - - /** - * This method will validate update page request data - * - * @param request Request - */ - public static void validateUpdatepage(Request request) { - if (request.getRequest().containsKey(JsonKey.PAGE_NAME) - && StringUtils.isEmpty( - (String) - (request.getRequest().get(JsonKey.PAGE_NAME) != null - ? request.getRequest().get(JsonKey.PAGE_NAME) - : ""))) { - throw new ProjectCommonException( - ResponseCode.pageNameRequired, - ResponseCode.pageNameRequired.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank( - (String) - (request.getRequest().get(JsonKey.ID) != null - ? request.getRequest().get(JsonKey.ID) - : ""))) { - throw new ProjectCommonException( - ResponseCode.pageIdRequired, - ResponseCode.pageIdRequired.getErrorMessage(), - ERROR_CODE); - } - } - - /** - * This method will validate bulk user upload requested data. - * - * @param reqObj Request - */ - public static void validateUploadUser(Map reqObj) { - if (StringUtils.isBlank((String) reqObj.get(JsonKey.ORGANISATION_ID)) - && (StringUtils.isBlank((String) reqObj.get(JsonKey.ORG_EXTERNAL_ID)) - || StringUtils.isBlank((String) reqObj.get(JsonKey.ORG_PROVIDER)))) { - throw new ProjectCommonException( - ResponseCode.mandatoryParamsMissing, - ProjectUtil.formatMessage( - ResponseCode.mandatoryParamsMissing.getErrorMessage(), - (ProjectUtil.formatMessage( - ResponseMessage.Message.OR_FORMAT, - JsonKey.ORGANISATION_ID, - ProjectUtil.formatMessage( - ResponseMessage.Message.AND_FORMAT, - JsonKey.ORG_EXTERNAL_ID, - JsonKey.ORG_PROVIDER)))), - ERROR_CODE); - } - if (null == reqObj.get(JsonKey.FILE)) { - throw new ProjectCommonException( - ResponseCode.mandatoryParamsMissing, - ProjectUtil.formatMessage( - ResponseCode.mandatoryParamsMissing.getErrorMessage(), JsonKey.FILE), - ERROR_CODE); - } - } - - /** - * courseId : Should be a valid courseId under EKStep. name : should not be null or empty - * enrolmentType: can have only following two values {"open","invite-only"} startDate : In - * yyyy-MM-DD format , and must be >= today date. endDate : In yyyy-MM-DD format and must be > - * startDate createdFor : List of valid organisation ids. this filed will be used in case of - * "invite-only" enrolmentType. for open type if createdFor values is coming then system will just - * save that value. mentors : List of user ids , who will work as a mentor. - * - * @param request - */ - public static void validateCreateBatchReq(Request request) { - if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.COURSE_ID))) { - throw new ProjectCommonException( - ResponseCode.invalidCourseId, - ResponseCode.invalidCourseId.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.NAME))) { - throw new ProjectCommonException( - ResponseCode.courseNameRequired, - ResponseCode.courseNameRequired.getErrorMessage(), - ERROR_CODE); - } - String enrolmentType = (String) request.getRequest().get(JsonKey.ENROLLMENT_TYPE); - validateEnrolmentType(enrolmentType); - String startDate = (String) request.getRequest().get(JsonKey.START_DATE); - String endDate = (String) request.getRequest().get(JsonKey.END_DATE); - validateStartDate(startDate); - validateEndDate(startDate, endDate); - - if (request.getRequest().containsKey(JsonKey.COURSE_CREATED_FOR) - && !(request.getRequest().get(JsonKey.COURSE_CREATED_FOR) instanceof List)) { - throw new ProjectCommonException( - ResponseCode.dataTypeError, - ResponseCode.dataTypeError.getErrorMessage(), - ERROR_CODE); - } - } - - private static boolean checkProgressStatus(int status) { - for (ProgressStatus pstatus : ProgressStatus.values()) { - if (pstatus.getValue() == status) { - return true; - } - } - return false; - } - - public static void validateUpdateCourseBatchReq(Request request) { - - if (null != request.getRequest().get(JsonKey.STATUS)) { - boolean status = validateBatchStatus(request); - if (!status) { - throw new ProjectCommonException( - ResponseCode.progressStatusError, - ResponseCode.progressStatusError.getErrorMessage(), - ERROR_CODE); - } - } - if (request.getRequest().containsKey(JsonKey.NAME) - && StringUtils.isEmpty((String) request.getRequest().get(JsonKey.NAME))) { - throw new ProjectCommonException( - ResponseCode.courseNameRequired, - ResponseCode.courseNameRequired.getErrorMessage(), - ERROR_CODE); - } - if (request.getRequest().containsKey(JsonKey.ENROLLMENT_TYPE)) { - String enrolmentType = (String) request.getRequest().get(JsonKey.ENROLLMENT_TYPE); - validateEnrolmentType(enrolmentType); - } - String startDate = (String) request.getRequest().get(JsonKey.START_DATE); - String endDate = (String) request.getRequest().get(JsonKey.END_DATE); - - validateUpdateBatchStartDate(startDate); - validateEndDate(startDate, endDate); - - boolean bool = validateDateWithTodayDate(endDate); - if (!bool) { - throw new ProjectCommonException( - ResponseCode.invalidBatchEndDateError, - ResponseCode.invalidBatchEndDateError.getErrorMessage(), - ERROR_CODE); - } - - validateUpdateBatchEndDate(request); - if (request.getRequest().containsKey(JsonKey.COURSE_CREATED_FOR) - && !(request.getRequest().get(JsonKey.COURSE_CREATED_FOR) instanceof List)) { - throw new ProjectCommonException( - ResponseCode.dataTypeError, - ResponseCode.dataTypeError.getErrorMessage(), - ERROR_CODE); - } - - if (request.getRequest().containsKey(JsonKey.MENTORS) - && !(request.getRequest().get(JsonKey.MENTORS) instanceof List)) { - throw new ProjectCommonException( - ResponseCode.dataTypeError, - ResponseCode.dataTypeError.getErrorMessage(), - ERROR_CODE); - } - } - - private static void validateUpdateBatchStartDate(String startDate) { - if (StringUtils.isNotBlank(startDate)) { - try { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - format.parse(startDate); - } catch (Exception e) { - throw new ProjectCommonException( - ResponseCode.dateFormatError, - ResponseCode.dateFormatError.getErrorMessage(), - ERROR_CODE); - } - } else { - throw new ProjectCommonException( - ResponseCode.courseBatchStartDateRequired, - ResponseCode.courseBatchStartDateRequired.getErrorMessage(), - ERROR_CODE); - } - } - - private static boolean validateBatchStatus(Request request) { - boolean status = false; - try { - status = checkProgressStatus(Integer.parseInt("" + request.getRequest().get(JsonKey.STATUS))); - - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - return status; - } - - private static void validateUpdateBatchEndDate(Request request) { - - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - String startDate = (String) request.getRequest().get(JsonKey.START_DATE); - String endDate = (String) request.getRequest().get(JsonKey.END_DATE); - format.setLenient(false); - if (StringUtils.isNotBlank(endDate) && StringUtils.isNotBlank(startDate)) { - Date batchStartDate = null; - Date batchEndDate = null; - try { - batchStartDate = format.parse(startDate); - batchEndDate = format.parse(endDate); - Calendar cal1 = Calendar.getInstance(); - Calendar cal2 = Calendar.getInstance(); - cal1.setTime(batchStartDate); - cal2.setTime(batchEndDate); - } catch (Exception e) { - throw new ProjectCommonException( - ResponseCode.dateFormatError, - ResponseCode.dateFormatError.getErrorMessage(), - ERROR_CODE); - } - if (batchEndDate.before(batchStartDate)) { - throw new ProjectCommonException( - ResponseCode.invalidBatchEndDateError, - ResponseCode.invalidBatchEndDateError.getErrorMessage(), - ERROR_CODE); - } - } - } - - private static boolean validateDateWithTodayDate(String date) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - format.setLenient(false); - try { - if (StringUtils.isNotEmpty(date)) { - Date reqDate = format.parse(date); - Date todayDate = format.parse(format.format(new Date())); - Calendar cal1 = Calendar.getInstance(); - Calendar cal2 = Calendar.getInstance(); - cal1.setTime(reqDate); - cal2.setTime(todayDate); - if (reqDate.before(todayDate)) { - return false; - } - } - } catch (Exception e) { - throw new ProjectCommonException( - ResponseCode.dateFormatError, - ResponseCode.dateFormatError.getErrorMessage(), - ERROR_CODE); - } - return true; - } - - /** @param enrolmentType */ - public static void validateEnrolmentType(String enrolmentType) { - if (StringUtils.isBlank(enrolmentType)) { - throw new ProjectCommonException( - ResponseCode.enrolmentTypeRequired, - ResponseCode.enrolmentTypeRequired.getErrorMessage(), - ERROR_CODE); - } - if (!(ProjectUtil.EnrolmentType.open.getVal().equalsIgnoreCase(enrolmentType) - || ProjectUtil.EnrolmentType.inviteOnly.getVal().equalsIgnoreCase(enrolmentType))) { - throw new ProjectCommonException( - ResponseCode.enrolmentIncorrectValue, - ResponseCode.enrolmentIncorrectValue.getErrorMessage(), - ERROR_CODE); - } - } - - /** @param startDate */ - private static void validateStartDate(String startDate) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - format.setLenient(false); - if (StringUtils.isBlank(startDate)) { - throw new ProjectCommonException( - ResponseCode.courseBatchStartDateRequired, - ResponseCode.courseBatchStartDateRequired.getErrorMessage(), - ERROR_CODE); - } - try { - Date batchStartDate = format.parse(startDate); - Date todayDate = format.parse(format.format(new Date())); - Calendar cal1 = Calendar.getInstance(); - Calendar cal2 = Calendar.getInstance(); - cal1.setTime(batchStartDate); - cal2.setTime(todayDate); - if (batchStartDate.before(todayDate)) { - throw new ProjectCommonException( - ResponseCode.courseBatchStartDateError, - ResponseCode.courseBatchStartDateError.getErrorMessage(), - ERROR_CODE); - } - } catch (ProjectCommonException e) { - throw e; - } catch (Exception e) { - throw new ProjectCommonException( - ResponseCode.dateFormatError, - ResponseCode.dateFormatError.getErrorMessage(), - ERROR_CODE); - } - } - - private static void validateEndDate(String startDate, String endDate) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - format.setLenient(false); - Date batchEndDate = null; - Date batchStartDate = null; - try { - if (StringUtils.isNotEmpty(endDate)) { - batchEndDate = format.parse(endDate); - batchStartDate = format.parse(startDate); - } - } catch (Exception e) { - throw new ProjectCommonException( - ResponseCode.dateFormatError, - ResponseCode.dateFormatError.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isNotEmpty(endDate) && batchStartDate.getTime() >= batchEndDate.getTime()) { - throw new ProjectCommonException( - ResponseCode.endDateError, - ResponseCode.endDateError.getErrorMessage(), - ERROR_CODE); - } - } - - public static void validateSyncRequest(Request request) { - String operation = (String) request.getRequest().get(JsonKey.OPERATION_FOR); - if ((null != operation) && (!operation.equalsIgnoreCase("keycloak"))) { - if (request.getRequest().get(JsonKey.OBJECT_TYPE) == null) { - throw new ProjectCommonException( - ResponseCode.dataTypeError, - ResponseCode.dataTypeError.getErrorMessage(), - ERROR_CODE); - } - List list = - new ArrayList<>( - Arrays.asList( - new String[] { - JsonKey.USER, JsonKey.ORGANISATION, JsonKey.BATCH, JsonKey.USER_COURSE - })); - if (!list.contains(request.getRequest().get(JsonKey.OBJECT_TYPE))) { - throw new ProjectCommonException( - ResponseCode.invalidObjectType, - ResponseCode.invalidObjectType.getErrorMessage(), - ERROR_CODE); - } - } - } - - public static void validateUpdateSystemSettingsRequest(Request request) { - List list = - new ArrayList<>( - Arrays.asList( - PropertiesCache.getInstance() - .getProperty("system_settings_properties") - .split(","))); - for (String str : request.getRequest().keySet()) { - if (!list.contains(str)) { - throw new ProjectCommonException( - ResponseCode.invalidPropertyError, - MessageFormat.format(ResponseCode.invalidPropertyError.getErrorMessage(), str), - ERROR_CODE); - } - } - } - - public static void validateSendMail(Request request) { - if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.SUBJECT))) { - throw new ProjectCommonException( - ResponseCode.emailSubjectError, - ResponseCode.emailSubjectError.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.BODY))) { - throw new ProjectCommonException( - ResponseCode.emailBodyError, - ResponseCode.emailBodyError.getErrorMessage(), - ERROR_CODE); - } - if (CollectionUtils.isEmpty((List) (request.getRequest().get(JsonKey.RECIPIENT_EMAILS))) - && CollectionUtils.isEmpty( - (List) (request.getRequest().get(JsonKey.RECIPIENT_USERIDS))) - && MapUtils.isEmpty( - (Map) (request.getRequest().get(JsonKey.RECIPIENT_SEARCH_QUERY))) - && CollectionUtils.isEmpty( - (List) (request.getRequest().get(JsonKey.RECIPIENT_PHONES)))) { - throw new ProjectCommonException( - ResponseCode.mandatoryParamsMissing, - MessageFormat.format( - ResponseCode.mandatoryParamsMissing.getErrorMessage(), - StringFormatter.joinByOr( - StringFormatter.joinByComma( - JsonKey.RECIPIENT_EMAILS, - JsonKey.RECIPIENT_USERIDS, - JsonKey.RECIPIENT_PHONES), - JsonKey.RECIPIENT_SEARCH_QUERY)), - ERROR_CODE); - } - } - - public static void validateFileUpload(Request reqObj) { - - if (StringUtils.isBlank((String) reqObj.get(JsonKey.CONTAINER))) { - throw new ProjectCommonException( - ResponseCode.storageContainerNameMandatory, - ResponseCode.storageContainerNameMandatory.getErrorMessage(), - ERROR_CODE); - } - } - - /** @param reqObj */ - public static void validateCreateOrgType(Request reqObj) { - if (StringUtils.isBlank((String) reqObj.getRequest().get(JsonKey.NAME))) { - throw createExceptionInstance(ResponseCode.orgTypeMandatory); - } - } - - /** @param reqObj */ - public static void validateUpdateOrgType(Request reqObj) { - if (StringUtils.isBlank((String) reqObj.getRequest().get(JsonKey.NAME))) { - throw createExceptionInstance(ResponseCode.orgTypeMandatory); - } - if (StringUtils.isBlank((String) reqObj.getRequest().get(JsonKey.ID))) { - throw createExceptionInstance(ResponseCode.orgTypeIdRequired); - } - } - - /** - * Method to validate not for userId, title, note, courseId, contentId and tags - * - * @param request - */ - @SuppressWarnings("rawtypes") - public static void validateNote(Request request) { - if (StringUtils.isBlank((String) request.get(JsonKey.USER_ID))) { - throw new ProjectCommonException( - ResponseCode.userIdRequired, - ResponseCode.userIdRequired.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank((String) request.get(JsonKey.TITLE))) { - throw new ProjectCommonException( - ResponseCode.titleRequired, - ResponseCode.titleRequired.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank((String) request.get(JsonKey.NOTE))) { - throw new ProjectCommonException( - ResponseCode.noteRequired, - ResponseCode.noteRequired.getErrorMessage(), - ERROR_CODE); - } - if (StringUtils.isBlank((String) request.get(JsonKey.CONTENT_ID)) - && StringUtils.isBlank((String) request.get(JsonKey.COURSE_ID))) { - throw new ProjectCommonException( - ResponseCode.contentIdError, - ResponseCode.contentIdError.getErrorMessage(), - ERROR_CODE); - } - if (request.getRequest().containsKey(JsonKey.TAGS) - && ((request.getRequest().get(JsonKey.TAGS) instanceof List) - && ((List) request.getRequest().get(JsonKey.TAGS)).isEmpty())) { - throw new ProjectCommonException( - ResponseCode.invalidTags, - ResponseCode.invalidTags.getErrorMessage(), - ERROR_CODE); - } else if (request.getRequest().get(JsonKey.TAGS) instanceof String) { - throw new ProjectCommonException( - ResponseCode.invalidTags, - ResponseCode.invalidTags.getErrorMessage(), - ERROR_CODE); - } - } - - /** - * Method to validate noteId - * - * @param noteId - */ - public static void validateNoteId(String noteId) { - if (StringUtils.isBlank(noteId)) { - throw createExceptionInstance(ResponseCode.invalidNoteId); - } - } - - /** - * Method to validate - * - * @param request - */ - public static void validateRegisterClient(Request request) { - - if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.CLIENT_NAME))) { - throw createExceptionInstance(ResponseCode.invalidClientName); - } - } - - /** - * Method to validate the request for updating the client key - * - * @param clientId - * @param masterAccessToken - */ - public static void validateUpdateClientKey(String clientId, String masterAccessToken) { - validateClientId(clientId); - if (StringUtils.isBlank(masterAccessToken)) { - throw createExceptionInstance(ResponseCode.invalidRequestData); - } - } - - /** - * Method to validate the request for updating the client key - * - * @param id - * @param type - */ - public static void validateGetClientKey(String id, String type) { - validateClientId(id); - if (StringUtils.isBlank(type)) { - throw createExceptionInstance(ResponseCode.invalidRequestData); - } - } - - /** - * Method to validate clientId. - * - * @param clientId - */ - public static void validateClientId(String clientId) { - if (StringUtils.isBlank(clientId)) { - throw createExceptionInstance(ResponseCode.invalidClientId); - } - } - - /** - * Method to validate notification request data. - * - * @param request Request - */ - @SuppressWarnings("unchecked") - public static void validateSendNotification(Request request) { - if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.TO))) { - throw createExceptionInstance(ResponseCode.invalidTopic); - } - if (request.getRequest().get(JsonKey.DATA) == null - || !(request.getRequest().get(JsonKey.DATA) instanceof Map) - || ((Map) request.getRequest().get(JsonKey.DATA)).size() == 0) { - throw createExceptionInstance(ResponseCode.invalidTopicData); - } - - if (StringUtils.isBlank((String) request.getRequest().get(JsonKey.TYPE))) { - throw createExceptionInstance(ResponseCode.invalidNotificationType); - } - if (!(JsonKey.FCM.equalsIgnoreCase((String) request.getRequest().get(JsonKey.TYPE)))) { - throw createExceptionInstance(ResponseCode.notificationTypeSupport); - } - } - - @SuppressWarnings("rawtypes") - public static void validateGetUserCount(Request request) { - if (!validateListType(request, JsonKey.LOCATION_IDS)) { - throw createDataTypeException( - ResponseCode.dataTypeError, JsonKey.LOCATION_IDS, JsonKey.LIST); - } - if (null == request.getRequest().get(JsonKey.LOCATION_IDS) - && ((List) request.getRequest().get(JsonKey.LOCATION_IDS)).isEmpty()) { - throw createExceptionInstance(ResponseCode.locationIdRequired); - } - - if (!validateBooleanType(request, JsonKey.USER_LIST_REQ)) { - throw createDataTypeException( - ResponseCode.dataTypeError, JsonKey.USER_LIST_REQ, "Boolean"); - } - - if (null != request.getRequest().get(JsonKey.USER_LIST_REQ) - && (Boolean) request.getRequest().get(JsonKey.USER_LIST_REQ)) { - throw createExceptionInstance(ResponseCode.functionalityMissing); - } - - if (!validateBooleanType(request, JsonKey.ESTIMATED_COUNT_REQ)) { - throw createDataTypeException( - ResponseCode.dataTypeError, JsonKey.ESTIMATED_COUNT_REQ, "Boolean"); - } - - if (null != request.getRequest().get(JsonKey.ESTIMATED_COUNT_REQ) - && (Boolean) request.getRequest().get(JsonKey.ESTIMATED_COUNT_REQ)) { - throw createExceptionInstance(ResponseCode.functionalityMissing); - } - } - - /** - * if the request contains that key and key is not instance of List then it will return false. - * other cases it will return true. - * - * @param request Request - * @param key String - * @return boolean - */ - private static boolean validateListType(Request request, String key) { - return !(request.getRequest().containsKey(key) - && null != request.getRequest().get(key) - && !(request.getRequest().get(key) instanceof List)); - } - - /** - * If the request contains the key and key value is not Boolean type then it will return false , - * for any other case it will return true. - * - * @param request Request - * @param key String - * @return boolean - */ - private static boolean validateBooleanType(Request request, String key) { - return !(request.getRequest().containsKey(key) - && null != request.getRequest().get(key) - && !(request.getRequest().get(key) instanceof Boolean)); - } - - private static ProjectCommonException createDataTypeException( - ResponseCode responseCode, String key1, String key2) { - return new ProjectCommonException( - responseCode, - ProjectUtil.formatMessage( - responseCode.getErrorMessage(), key1, key2), - ERROR_CODE); - } - - private static ProjectCommonException createExceptionInstance(ResponseCode responseCode) { - return new ProjectCommonException( - responseCode, - responseCode.getErrorMessage(), - ERROR_CODE); - } -} From 4a4bee10ac62845003fd8af5687d30c433ff755f Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Thu, 6 Aug 2026 11:02:29 +0530 Subject: [PATCH 23/30] test(viewer): regression tests for C1 (phantom-row guard) and C2 (contentstatus merge) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C1 fix and the courseId/batchId revert had left the suite red (tests were not re-run): touchEnrolmentAccess's getRecordByIdentifier was unstubbed, and the aggregator guard tests sent stale collectionId/contextId keys. Restore green + pin the two fixes. - ViewConsumptionActorTest: stub the enrolment read in every write-op test; add touchEnrolmentAccess cases — no stamp when the enrolment is absent (no updateRecordV2 -> phantom-row guard), stamp exactly once when present. - ViewerAggregatorActor: extract the contentstatus merge into a pure companion mergeContentStatus (updateRecordV2 replaces the whole column, so a root-keyed rollup must merge, not clobber). - ViewerAggregatorActorTest: fix guard-test request keys; add mergeContentStatus cases (preserve existing leaves, fresh wins on conflict, null-tolerant). 19/19 green. --- .../viewer/actor/ViewerAggregatorActor.scala | 21 +++++--- .../actor/ViewConsumptionActorTest.scala | 49 ++++++++++++++++++- .../actor/ViewerAggregatorActorTest.scala | 30 ++++++++++-- 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index 72bb9a71..fc7ed6ea 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -317,12 +317,11 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { val status = activityAggUtil.getCompletionStatus(completedCount, required) val pct = activityAggUtil.getCompletionPercentage(completedCount, required) val currentStatus = Option(row.get("status")).map(_.asInstanceOf[Number].intValue()).getOrElse(0) - val nodeContentStatus = requiredLeaves.flatMap(l => contentStatusMap.get(l).map(cs => l -> Integer.valueOf(cs.status))).toMap - // contentstatus is a full-column replace in updateRecordV2 — merge into the row's existing map so - // leaves not in this (root-keyed) read aren't clobbered. - val mergedContentStatus = new util.HashMap[String, AnyRef]() - Option(row.get("contentStatus")).foreach(m => mergedContentStatus.putAll(m.asInstanceOf[util.Map[String, AnyRef]])) - nodeContentStatus.foreach { case (k, v) => mergedContentStatus.put(k, v) } + val nodeContentStatus: Map[String, AnyRef] = + requiredLeaves.flatMap(l => contentStatusMap.get(l).map(cs => l -> Integer.valueOf(cs.status).asInstanceOf[AnyRef])).toMap + // updateRecordV2 replaces the whole contentstatus column — merge into the row's existing map so a + // root-keyed rollup that only sees some leaves doesn't clobber the rest (see mergeContentStatus). + val mergedContentStatus = ViewerAggregatorActor.mergeContentStatus(row.get("contentStatus"), nodeContentStatus) val selectMap = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", nodeId); put("batchid", nodeCtx) }} @@ -385,6 +384,16 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { } object ViewerAggregatorActor { + // Merge freshly computed per-leaf statuses into the enrolment row's existing contentstatus map instead of + // replacing it: updateRecordV2 overwrites the whole column, so a root-keyed rollup that only sees some + // leaves must not wipe the rest (C2). `existing` may be null; fresh values win on key conflicts. + private[actor] def mergeContentStatus(existing: AnyRef, fresh: Map[String, AnyRef]): java.util.Map[String, AnyRef] = { + val merged = new java.util.HashMap[String, AnyRef]() + Option(existing).foreach(m => merged.putAll(m.asInstanceOf[java.util.Map[String, AnyRef]])) + fresh.foreach { case (k, v) => merged.put(k, v) } + merged + } + // JVM-wide memo of enrolments whose (empty) LP optionality is computed, so we don't recompute each pass. // ponytail: unbounded set, entries live for the process lifetime; add a size cap / TTL only if it grows. private val optionalityDone: java.util.Set[String] = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala index f403767d..a43f4e6c 100644 --- a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewConsumptionActorTest.scala @@ -20,6 +20,10 @@ import scala.concurrent.duration.FiniteDuration * (submit no-events branch + read). CassandraOperation is mocked via the setCassandraOperation seam; * the aggregator is a stub actor that replies immediately so the sync ask in viewEnd/viewAssess * returns fast. Scoring math itself is covered by AssessmentServiceSpec, not re-tested here. + * + * Every write op also calls touchEnrolmentAccess, which reads the enrolment (getRecordByIdentifier) + * and only stamps it (updateRecordV2) when it EXISTS — so each test stubs that read. The two + * "touchEnrolmentAccess" cases pin that guard (C1: no phantom enrolment row on an unenrolled view). */ class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactory { @@ -44,6 +48,19 @@ class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactor put("status", Integer.valueOf(status)) }} + private def enrolmentRow: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]() {{ + put("userid", "u1"); put("courseId", "c1"); put("batchId", "b1"); put("status", Integer.valueOf(1)) + }} + + // touchEnrolmentAccess's enrolment read; `result` decides whether the row is stamped. + private def stubEnrolmentRead(ops: CassandraOperation, result: Response) = + (ops.getRecordByIdentifier(_: String, _: String, _: Object, _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(result) + + private def expectEnrolmentStamp(ops: CassandraOperation) = + (ops.updateRecordV2(_: String, _: String, _: util.Map[String, AnyRef], _: util.Map[String, AnyRef], _: Boolean, _: RequestContext)) + .expects(*, *, *, *, *, *).returns(new Response()).once() + private def callActor(request: Request, props: Props): Response = { val probe = new TestKit(system) val actorRef = system.actorOf(props) @@ -54,7 +71,7 @@ class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactor private def viewRequest(op: String): Request = { val req = new Request req.setOperation(op) - req.put("userId", "u1"); req.put("collectionId", "c1"); req.put("contextId", "b1"); req.put("contentId", "ct1") + req.put("userId", "u1"); req.put("courseId", "c1"); req.put("batchId", "b1"); req.put("contentId", "ct1") req } @@ -64,6 +81,7 @@ class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactor .expects(*, *, *, *, *).returns(emptyRows) (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) val result = callActor(viewRequest("viewStart"), Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) result should not be null @@ -75,6 +93,7 @@ class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactor (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) .expects(*, *, *, *, *).returns(rowsWith(rows)) // no upsertRecord expectation -> a call would fail the strict mock + stubEnrolmentRead(ops, emptyRows) val result = callActor(viewRequest("viewStart"), Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) result should not be null @@ -87,6 +106,7 @@ class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactor .expects(*, *, *, *, *).returns(rowsWith(rows)) (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) val result = callActor(viewRequest("viewUpdate"), Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) result should not be null @@ -96,6 +116,32 @@ class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactor val ops = mock[CassandraOperation] (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) + val result = callActor(viewRequest("viewEnd"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + // C1: touchEnrolmentAccess must NOT fabricate an enrolment for an unenrolled/no-context view. + "touchEnrolmentAccess" should "not stamp the enrolment when none exists" in { + val ops = mock[CassandraOperation] + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) // no enrolment + // no updateRecordV2 expectation -> a stamp write would fail the strict mock (phantom-row guard) + val result = callActor(viewRequest("viewEnd"), + Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) + result should not be null + } + + // C1: when the enrolment DOES exist, its last-access is stamped exactly once. + "touchEnrolmentAccess" should "stamp the enrolment when it exists" in { + val ops = mock[CassandraOperation] + (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) + .expects(*, *, *, *).returns(new Response()).once() + val rows = new util.ArrayList[util.Map[String, AnyRef]](); rows.add(enrolmentRow) + stubEnrolmentRead(ops, rowsWith(rows)) // enrolment present + expectEnrolmentStamp(ops) val result = callActor(viewRequest("viewEnd"), Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) result should not be null @@ -116,6 +162,7 @@ class ViewConsumptionActorTest extends AnyFlatSpec with Matchers with MockFactor val ops = mock[CassandraOperation] (ops.upsertRecord(_: String, _: String, _: util.Map[String, AnyRef], _: RequestContext)) .expects(*, *, *, *).returns(new Response()).once() + stubEnrolmentRead(ops, emptyRows) val result = callActor(viewRequest("viewAssess"), Props(new ViewConsumptionActor(replyingAggregator).setCassandraOperation(ops))) result.getResult.get("ct1") shouldBe "SUCCESS" diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala index 3b63489b..0113d3e6 100644 --- a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala @@ -37,16 +37,16 @@ class ViewerAggregatorActorTest extends AnyFlatSpec with Matchers with MockFacto probe.expectMsgType[Response](FiniteDuration.apply(15, TimeUnit.SECONDS)) } - private def aggRequest(userId: String, collectionId: String): Request = { + private def aggRequest(userId: String, courseId: String): Request = { val req = new Request req.setOperation("aggregate") if (userId != null) req.put("userId", userId) - if (collectionId != null) req.put("collectionId", collectionId) - req.put("contextId", "b1") + if (courseId != null) req.put("courseId", courseId) + req.put("batchId", "b1") req } - "aggregate" should "skip and reply success when userId/collectionId are missing" in { + "aggregate" should "skip and reply success when userId/courseId are missing" in { val ops = mock[CassandraOperation] val hru = mock[HierarchyRelationsUtil] val cu = mock[CertificateUtil] @@ -67,4 +67,26 @@ class ViewerAggregatorActorTest extends AnyFlatSpec with Matchers with MockFacto val result = callActor(aggRequest("u1", "c1"), Props(new ViewerAggregatorActor().configure(ops, hru, cu))) result should not be null } + + // C2: contentstatus is a full-column replace in updateRecordV2, so the rollup must MERGE freshly computed + // leaf statuses into the enrolment row's existing map — a root-keyed read that sees only some leaves must + // not wipe the others. Tests the extracted pure merge directly (the full rollup is an integration concern). + "mergeContentStatus" should "preserve existing leaves and add/overwrite the fresh ones" in { + val existing = new util.HashMap[String, AnyRef]() {{ + put("leaf-a", Integer.valueOf(2)) // completed earlier, not in this pass + put("leaf-b", Integer.valueOf(1)) // in-progress, gets overwritten below + }} + val fresh = Map[String, AnyRef]("leaf-b" -> Integer.valueOf(2), "leaf-c" -> Integer.valueOf(2)) + val merged = ViewerAggregatorActor.mergeContentStatus(existing, fresh) + merged.get("leaf-a") shouldBe Integer.valueOf(2) // preserved (not clobbered) + merged.get("leaf-b") shouldBe Integer.valueOf(2) // fresh wins on conflict + merged.get("leaf-c") shouldBe Integer.valueOf(2) // added + merged.size() shouldBe 3 + } + + "mergeContentStatus" should "tolerate a null existing map" in { + val merged = ViewerAggregatorActor.mergeContentStatus(null, Map[String, AnyRef]("leaf-a" -> Integer.valueOf(1))) + merged.get("leaf-a") shouldBe Integer.valueOf(1) + merged.size() shouldBe 1 + } } From 219d66c31ba50c80ea53d59c04c39ccf32eafe2a Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Fri, 7 Aug 2026 12:23:08 +0530 Subject: [PATCH 24/30] feat(lp): single enrol path for LP + child-batch cert inheritance; courseId/batchId cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CourseEnrolmentActor: remove systemEnroll (and its onReceive route); LP progression now uses the standard `enrol` op. No parallel enrol method. - ViewerAggregatorActor: internalEnrol dispatches `enrol` with requestId="system" (recorded as addedBy via context REQUEST_ID); re-enrol safety stays the caller's (advanceLp only enrols courses absent from the snapshot). Skill-derivation stub comments point at the framework last-category contract (design §6), not se_skills. - CourseBatchManagementActor: nested-trackable child batches inherit the LP root batch's cert_templates (strip removed) -> each child course issues the LP cert. - ActivityAggregatorActor: viewer-dispatch payload uses courseId/batchId (was the leftover collectionId/contextId, which the viewer never reads). - ContentConsumptionActor: drop the leftover collectionId alias from content-state read (courseId is canonical); update CourseConsumptionActorTest expectation. --- .../actor/ActivityAggregatorActor.scala | 8 +++---- .../CourseBatchManagementActor.java | 8 ++----- .../enrolments/ContentConsumptionActor.scala | 1 - .../enrolments/CourseEnrolmentActor.scala | 24 ------------------- .../CourseConsumptionActorTest.scala | 2 +- .../viewer/actor/ViewerAggregatorActor.scala | 14 +++++------ 6 files changed, 14 insertions(+), 43 deletions(-) diff --git a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala index 534f4d5d..c398cb80 100644 --- a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala +++ b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/actor/ActivityAggregatorActor.scala @@ -109,16 +109,16 @@ class ActivityAggregatorActor extends BaseEnrolmentActor { val (op, api) = if (status >= 2) ("viewEnd", "/v1/view/end") else ("viewStart", "/v1/view/start") dispatch("view-consumption-actor", api, op, new util.HashMap[String, AnyRef]() {{ put("contentId", c.get(JsonKey.CONTENT_ID)) - put("collectionId", courseId) - put("contextId", batchId) + put("courseId", courseId) + put("batchId", batchId) put(JsonKey.USER_ID, userId) Option(c.get("progressdetails")).orElse(Option(c.get("progressDetails"))).foreach(pd => put("progressDetails", pd)) }}) } } else { dispatch("viewer-aggregator-actor", "/v1/view/agg", "aggregate", new util.HashMap[String, AnyRef]() {{ - put("collectionId", courseId) - put("contextId", batchId) + put("courseId", courseId) + put("batchId", batchId) put(JsonKey.USER_ID, userId) }}) } diff --git a/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java b/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java index 4f6a97c0..6cd22b53 100644 --- a/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java +++ b/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java @@ -167,12 +167,8 @@ private void triggerChildBatchCreation(Request parent, String parentCourseId, St Map childReq = new HashMap<>(parent.getRequest()); childReq.put(JsonKey.COURSE_ID, childCourseId); childReq.put(JsonKey.BATCH_ID, childBatchId); - // Child batches must NOT inherit the LP root's certificate template (that template is the LP cert). - // Course certs are controlled by `courseCertificates` (per-LP, default off); when enabled a course's - // own cert template is attached separately. Always strip the inherited template so the default is - // "LP cert only" and a course never wrongly issues the LP certificate. - childReq.remove("certTemplates"); - childReq.remove("cert_templates"); + // Child batches inherit the LP root's certificate template (carried through via parent.getRequest()), + // so every trackable child issues the LP certificate on its own completion. Intentionally NOT stripped. child.setRequest(childReq); logger.info(ctx, "triggerChildBatchCreation: creating batch " + childBatchId + " for course " + childCourseId); self().tell(child, ActorRef.noSender()); diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala index 6f004730..41f239b0 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/ContentConsumptionActor.scala @@ -638,7 +638,6 @@ class ContentConsumptionActor @Inject() ( if(CollectionUtils.isNotEmpty(contentsConsumed)) { val filteredContents = contentsConsumed.map(m => { ProjectUtil.removeUnwantedFields(m, JsonKey.DATE_TIME, JsonKey.USER_ID, JsonKey.ADDED_BY, JsonKey.LAST_UPDATED_TIME, JsonKey.OLD_LAST_ACCESS_TIME, JsonKey.OLD_LAST_UPDATED_TIME, JsonKey.OLD_LAST_COMPLETED_TIME) - m.put(JsonKey.COLLECTION_ID, m.getOrDefault(JsonKey.COURSE_ID, "")) jsonFields.foreach(field => if(m.get(field) != null) m.put(field, mapper.readTree(m.get(field).asInstanceOf[String])) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index dcfd5f12..434cdecc 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -74,7 +74,6 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c request.getOperation match { case "enrol" => enroll(request) - case "systemEnrol" => systemEnroll(request) case "unenrol" => unEnroll(request) case "listEnrol" => list(request) case _ => ProjectCommonException.throwClientErrorException(ResponseCode.invalidRequestData, @@ -99,28 +98,6 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c generateTelemetryAudit(userId, courseId, batchId, data, "enrol", JsonKey.CREATE, request.getContext) notifyUser(userId, batchData, JsonKey.ADD) } - - /** - * System-driven enrol (the `doEnrol` seam) — used by LP progression to open a course. - * Reuses the SAME verified write path as `enroll` (createUserEnrolmentMap + upsertEnrollment + - * cache-clear + telemetry audit), so LP auto-enrolments are first-class. Differences: `addedBy = - * system-lp`, notifications SUPPRESSED (no per-auto-enrol spam), no descendant fan-out (this is a - * single course), and idempotent (already-enrolled -> success no-op). Called via the - * ProgressionEnroller gateway: in-JVM (monolith) or HTTP (distributed). - */ - def systemEnroll(request: Request): Unit = { - val courseId: String = request.get(JsonKey.COURSE_ID).asInstanceOf[String] - val userId: String = request.get(JsonKey.USER_ID).asInstanceOf[String] - val batchId: String = request.get(JsonKey.BATCH_ID).asInstanceOf[String] - val enrolmentData: UserCourses = userCoursesDao.read(request.getRequestContext, userId, courseId, batchId) - if (null != enrolmentData) { sender().tell(successResponse(), self); return } // idempotent - val data: java.util.Map[String, AnyRef] = createUserEnrolmentMap(userId, courseId, batchId, enrolmentData, "system-lp") - upsertEnrollment(userId, courseId, batchId, data, true, request.getRequestContext) - if (isCacheEnabled) cacheUtil.delete(getCacheKey(userId)) - sender().tell(successResponse(), self) - generateTelemetryAudit(userId, courseId, batchId, data, "enrol", JsonKey.CREATE, request.getContext) - // notifications intentionally suppressed for system-lp; no enrolTrackableDescendants (single course). - } def unEnroll(request:Request): Unit = { @@ -187,7 +164,6 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c enrolment.put(JsonKey.LEAF_NODE_COUNT, courseContent.get(JsonKey.LEAF_NODE_COUNT)) enrolment.put(JsonKey.COURSE_LOGO_URL, courseContent.get(JsonKey.APP_ICON)) enrolment.put(JsonKey.CONTENT_ID, enrolment.get(JsonKey.COURSE_ID)) - enrolment.put(JsonKey.COLLECTION_ID, enrolment.get(JsonKey.COURSE_ID)) enrolment.put(JsonKey.CONTENT, courseContent) enrolment }).toList.asJava diff --git a/modules/lms/course-mw/enrolment-actor/src/test/scala/org/sunbird/enrolments/CourseConsumptionActorTest.scala b/modules/lms/course-mw/enrolment-actor/src/test/scala/org/sunbird/enrolments/CourseConsumptionActorTest.scala index 6fc82a29..51e3e76d 100644 --- a/modules/lms/course-mw/enrolment-actor/src/test/scala/org/sunbird/enrolments/CourseConsumptionActorTest.scala +++ b/modules/lms/course-mw/enrolment-actor/src/test/scala/org/sunbird/enrolments/CourseConsumptionActorTest.scala @@ -295,7 +295,7 @@ class CourseConsumptionActorTest extends FlatSpec with Matchers with MockFactory ((keyspace: _root_.scala.Predef.String, table: _root_.scala.Predef.String, filters: _root_.java.util.Map[_root_.scala.Predef.String, AnyRef], fields: _root_.java.util.List[_root_.scala.Predef.String], requestContext: RequestContext) => cassandraOperation.getRecords(keyspace, table, filters, fields, requestContext)).expects(*, *, *, *, *).returns(response) val result = callActor(getStateReadRequestWithProgressField(), Props(new ContentConsumptionActor(mockActivityAggregatorActor, mockAssessmentAggregatorActor).setCassandraOperation(cassandraOperation, false))) - result.getResult().get("response").toString.shouldEqual("[{progressDetails={key1=val1, key2=val2}, contentId=do_456, batchId=0123, courseId=do_123, collectionId=do_123, progressdetails={}}]") + result.getResult().get("response").toString.shouldEqual("[{progressDetails={key1=val1, key2=val2}, contentId=do_456, batchId=0123, courseId=do_123, progressdetails={}}]") assert(null != result) } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index fc7ed6ea..0899b9a0 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -193,10 +193,11 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { private def policyOf(rootId: String, ctx: RequestContext): String = "Strict" // VERIFY-ON-DEPLOY: assessment detection. Needs a Practice-Question-Set child via /v3/search or content_hierarchy. private def isAssessmentCourse(courseId: String, ctx: RequestContext): Boolean = false - // VERIFY-ON-DEPLOY: per-course skills (se_skills) + assessment flag via /v3/search. + // VERIFY-ON-DEPLOY: per-course skills + assessment flag via /v3/search. Skills = framework last-category + // terms (se_Ids), not an se_skills field (see design §6). private def courseMeta(trackable: List[String], ctx: RequestContext): Map[String, (Set[String], Boolean)] = trackable.map(c => c -> (Set.empty[String], isAssessmentCourse(c, ctx))).toMap - // VERIFY-ON-DEPLOY: derive from best-attempt assessment_aggregator scores × se_skills tags (skill achieved = all its questions correct). + // VERIFY-ON-DEPLOY: achieved = assessment_aggregator best attempts × question skill ids (all correct); ids = framework last-category terms (design §6). private def skillsFromAssessment(userId: String, rootId: String, courseId: String, ctx: RequestContext): Set[String] = Set.empty /** @@ -218,10 +219,8 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { } /** - * Internal (system-driven) enrol via the ProgressionEnroller gateway — the FULL enrol op (`doEnrol`), - * NOT a bare DAO write: it reuses CourseEnrolmentActor's verified write path (DB + cache + telemetry). - * Transport per `deployment_mode`: MONOLITH -> in-JVM message to the enrolment actor's `systemEnrol`; - * DISTRIBUTED -> HTTP to the enrolment service. Idempotent (`systemEnrol` no-ops if already enrolled). + * System-driven enrol via the standard `enrol` op. MONOLITH -> in-JVM message; DISTRIBUTED -> POST + * /v1/course/enroll. Re-enrol safe: advanceLp only enrols courses absent from the snapshot. */ private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) // default monolith @@ -229,7 +228,8 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { if (isMonolith) { val req = new Request() req.setRequestContext(ctx) - req.setOperation("systemEnrol") + req.setRequestId("system") // enrol stores this as addedBy (via context REQUEST_ID) + req.setOperation("enrol") req.put(JsonKey.USER_ID, userId); req.put(JsonKey.COURSE_ID, courseId); req.put(JsonKey.BATCH_ID, batchId) // VERIFY-ON-DEPLOY: bound path of the enrolment actor in the monolith actor system. val path = Option(ProjectUtil.getConfigValue("enrolment_actor_path")).filter(_.nonEmpty).getOrElse("/user/course-enrolment-actor") From d98c2750328e172f1b79aae3a1a9b5cd2b00c84a Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Mon, 10 Aug 2026 10:18:10 +0530 Subject: [PATCH 25/30] feat(viewer): state loggers across viewer flow + wire enrol->LP bootstrap - Consistent fixed-phrase loggers (stage=view/rollup/lp/summary, key=val) at each state transition: view start/update/end/access, rollup start/LP-detect/nodes/ node-complete->cert, lp optionality/level-opened/enrol/complete, summary ops. - CourseEnrolmentActor: log an enrol confirmation ("enrol: enrolled ..."). - Wire the enrol->LP bootstrap (design Step 3): fireLpBootstrap after a user enrol fires the viewer rollup once (monolith in-JVM tell to viewer-aggregator-actor; distributed HTTP /v1/view/agg), so advanceLp opens the first LP course at enrol time. Skips system-lp child enrols and no-ops when viewer disabled. enroll's core logic unchanged. - Fix stale "system-enrol requested" log wording in ViewerAggregatorActor. --- .../enrolments/CourseEnrolmentActor.scala | 27 +++++++++++++++++++ .../viewer/actor/ViewConsumptionActor.scala | 13 +++++---- .../viewer/actor/ViewerAggregatorActor.scala | 23 +++++++++++----- .../viewer/actor/ViewerSummaryActor.scala | 5 +++- 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala index 434cdecc..3c49a665 100644 --- a/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala +++ b/modules/lms/course-mw/enrolment-actor/src/main/scala/org/sunbird/enrolments/CourseEnrolmentActor.scala @@ -7,6 +7,7 @@ import org.apache.commons.lang3.StringUtils import org.sunbird.cache.util.RedisCacheUtil import org.sunbird.common.CassandraUtil import org.sunbird.exception.ProjectCommonException +import org.sunbird.http.HttpClientUtil import org.sunbird.response.Response import org.sunbird.common.ProjectUtil import org.sunbird.common.ProjectUtil.EnrolmentType @@ -90,6 +91,7 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c validateEnrolment(batchData, enrolmentData, true) val data: java.util.Map[String, AnyRef] = createUserEnrolmentMap(userId, courseId, batchId, enrolmentData, request.getContext.getOrDefault(JsonKey.REQUEST_ID, "").asInstanceOf[String]) upsertEnrollment(userId, courseId, batchId, data, (null == enrolmentData), request.getRequestContext) + logger.info(request.getRequestContext, s"enrol: enrolled | user=$userId course=$courseId batch=$batchId new=${null == enrolmentData}") if (isCacheEnabled) { logger.info(request.getRequestContext, "CourseEnrolmentActor :: enroll :: Deleting redis for key " + getCacheKey(userId)) cacheUtil.delete(getCacheKey(userId)) @@ -97,6 +99,31 @@ class CourseEnrolmentActor @Inject()(@Named("course-batch-notification-actor") c sender().tell(successResponse(), self) generateTelemetryAudit(userId, courseId, batchId, data, "enrol", JsonKey.CREATE, request.getContext) notifyUser(userId, batchData, JsonKey.ADD) + fireLpBootstrap(userId, courseId, batchId, request) + } + + /** + * Open the first LP course right after a USER enrol (design Step 3): fire the viewer rollup once so + * advanceLp runs at enrol time instead of only on first consumption. Skips system-lp child enrols + * (already driven by their rollup) and no-ops when the viewer is off. Fire-and-forget; monolith -> + * in-JVM to the aggregator, distributed -> HTTP /v1/view/agg. Mirrors ContentConsumptionActor's transport. + */ + private def fireLpBootstrap(userId: String, courseId: String, batchId: String, request: Request): Unit = { + val requestId = Option(request.getContext.get(JsonKey.REQUEST_ID)).map(_.toString).getOrElse("") + if ("system".equals(requestId)) return + if (!java.lang.Boolean.parseBoolean(ProjectUtil.getConfigValue("viewer_enabled"))) return + try { + if (!"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode"))) { + val agg = new Request(); agg.setRequestContext(request.getRequestContext); agg.setOperation("aggregate") + agg.put(JsonKey.USER_ID, userId); agg.put("courseId", courseId); agg.put("batchId", batchId) + context.actorSelection("/user/viewer-aggregator-actor").tell(agg, ActorRef.noSender) + } else { + val base = Option(ProjectUtil.getConfigValue("viewer_service_base_url")).filter(StringUtils.isNotBlank).getOrElse("http://viewer-service:9000") + val headers = new util.HashMap[String, String]() {{ put("Content-Type", "application/json") }} + HttpClientUtil.post(base + "/v1/view/agg", s"""{"request":{"userId":"$userId","courseId":"$courseId","batchId":"$batchId"}}""", headers, request.getRequestContext) + } + logger.info(request.getRequestContext, s"enrol: LP bootstrap fired | user=$userId course=$courseId batch=$batchId") + } catch { case ex: Exception => logger.error(request.getRequestContext, s"enrol: LP bootstrap failed: ${ex.getMessage}", ex) } } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala index 0176b34e..e784f6d7 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewConsumptionActor.scala @@ -54,13 +54,14 @@ class ViewConsumptionActor @Inject() ( // Cassandra), so without this guard a no-context/unenrolled view would fabricate a phantom enrolment row. val existing = cassandraOperation.getRecordByIdentifier(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, null, ctx) .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] - if (existing.isEmpty) return + if (existing.isEmpty) { logger.info(ctx, s"view: access skip(no-enrolment) | user=${key.get("userid")} course=${key.get("courseid")} batch=${key.get("batchid")}"); return } val updateMap = new util.HashMap[String, AnyRef]() {{ put("lastcontentaccesstime", new java.util.Date()) put("lastreadcontentid", key.get("contentid")) put("lastreadcontentstatus", Integer.valueOf(status)) }} cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) + logger.info(ctx, s"view: access stamped | user=${key.get("userid")} course=${key.get("courseid")} content=${key.get("contentid")} status=$status") } // Assessment scoring reuses the assessment-aggregator services in-process (same math + persistence @@ -198,8 +199,8 @@ class ViewConsumptionActor @Inject() ( row.put("last_access_time", ProjectUtil.getTimeStamp) row.put("last_updated_time", ProjectUtil.getTimeStamp) cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) - } - // present -> already started, no-op + logger.info(ctx, s"view: start inserted | user=${key.get("userid")} course=${key.get("courseid")} batch=${key.get("batchid")} content=${key.get("contentid")}") + } else logger.info(ctx, s"view: start noop(exists) | user=${key.get("userid")} content=${key.get("contentid")}") touchEnrolmentAccess(key, 1, ctx) sender().tell(successResponse(), self) } @@ -216,8 +217,8 @@ class ViewConsumptionActor @Inject() ( row.put("last_access_time", ProjectUtil.getTimeStamp) row.put("last_updated_time", ProjectUtil.getTimeStamp) cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) - } - // absent -> update only if exists (ignore); already completed -> revisit ignored + logger.info(ctx, s"view: update merged | user=${key.get("userid")} content=${key.get("contentid")}") + } else logger.info(ctx, s"view: update skip(absent-or-completed) | user=${key.get("userid")} content=${key.get("contentid")}") touchEnrolmentAccess(key, math.max(1, if (existing != null) statusOf(existing) else 1), ctx) sender().tell(successResponse(), self) } @@ -231,6 +232,7 @@ class ViewConsumptionActor @Inject() ( row.put("last_completed_time", ProjectUtil.getTimeStamp) row.put("last_updated_time", ProjectUtil.getTimeStamp) cassandraOperation.upsertRecord(consumptionDBInfo.getKeySpace, CONSUMPTION_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + logger.info(ctx, s"view: end completed | user=${key.get("userid")} course=${key.get("courseid")} batch=${key.get("batchid")} content=${key.get("contentid")}") touchEnrolmentAccess(key, 2, ctx) // Async rollup: fire-and-forget tell to the aggregator; respond immediately (does not wait). triggerAggregation(request, ctx) @@ -248,6 +250,7 @@ class ViewConsumptionActor @Inject() ( aggRequest.put("batchId", key.get("batchid")) // Async, fire-and-forget: the rollup + LP progression run in the background on the aggregator // (per-user serialized). The hot path does not wait for it — the change from before is ask -> tell. + logger.info(ctx, s"view: rollup triggered | user=${key.get("userid")} course=${key.get("courseid")} batch=${key.get("batchid")}") viewerAggregatorActor.tell(aggRequest, ActorRef.noSender) } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index 0899b9a0..59defa41 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -67,13 +67,15 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // trackablenodes non-empty => this root is a Learning Path (structural detection; §Step 2/5). val trackable = hierarchyRelationsUtil.getTrackableNodes(courseId, ctx) + logger.info(ctx, s"viewer.rollup: start | user=$userId course=$courseId batch=$batchId " + + (if (trackable.nonEmpty) s"LP detected n=${trackable.size}" else "not-an-LP")) // 1. Read this user's consumption for the (root) collection+context from viewer ucc, build status map val rows = readConsumption(userId, courseId, batchId, ctx) if (CollectionUtils.isEmpty(rows)) { // No consumption yet. For an LP, still advance (bootstrap: open the first required course). - if (trackable.nonEmpty) advanceLp(userId, courseId, batchId, trackable, ctx) - else logger.info(ctx, s"ViewerAggregatorActor: no consumption for userId=$userId courseId=$courseId") + if (trackable.nonEmpty) { logger.info(ctx, s"viewer.rollup: no-consumption -> LP bootstrap | user=$userId course=$courseId"); advanceLp(userId, courseId, batchId, trackable, ctx) } + else logger.info(ctx, s"viewer.rollup: no-consumption skip | user=$userId course=$courseId") return } val contentStatusMap: Map[String, ContentStatus] = activityAggUtil.getContentStatusFromContents(rows) @@ -111,6 +113,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // 5. Write user_activity_agg (frozen content_status + agg) for root + every node writeActivityAggregates(allAggs, ctx) + logger.info(ctx, s"viewer.rollup: nodes rolled-up n=${allAggs.size} | user=$userId course=$courseId batch=$batchId") // 6. Per-node progress: nodeId -> (completedCount, requiredLeaves) for root + every trackable ancestor. val nodeProgress = scala.collection.mutable.LinkedHashMap[String, (Int, List[String])]() @@ -160,14 +163,20 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { levels.find(l => !levelComplete(l)).foreach { level => val courses = ProgressionPolicy.coursesOfLevel(level, trackable, ancestorsOf, rootId) val nextRequired = courses.filterNot(optional.contains).find(c => !courseComplete(c)) - (courses.filter(optional.contains) ++ nextRequired.toList).foreach { c => + val toOpen = courses.filter(optional.contains) ++ nextRequired.toList + logger.info(ctx, s"viewer.lp: level opened | user=$userId root=$rootId level=$level open=[${toOpen.mkString(",")}]") + toOpen.foreach { c => val childBatch = childBatchOf(c) if (!status.contains((c, childBatch))) internalEnrol(userId, c, childBatch, ctx) + else logger.info(ctx, s"viewer.lp: enrol skip(already) | user=$userId course=$c batch=$childBatch") } } // LP completion = every level complete -> credit durable skills (once; creditSkills no-ops if nothing new). - if (levels.nonEmpty && levels.forall(levelComplete)) creditSkills(userId, rootId, trackable, ctx) + if (levels.nonEmpty && levels.forall(levelComplete)) { + logger.info(ctx, s"viewer.lp: complete | user=$userId root=$rootId") + creditSkills(userId, rootId, trackable, ctx) + } } private def ensureOptionalityComputed(userId: String, rootId: String, batchId: String, trackable: List[String], @@ -175,7 +184,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // Compute once (§Step 4); optionalityComputed remembers empty results without a DB column. if (optionalityComputed(userId, rootId, batchId, ctx)) return val policy = policyOf(rootId, ctx) - if (policy.equalsIgnoreCase("Strict") || trackable.isEmpty) { writeOptionalNodes(userId, rootId, batchId, Set.empty, ctx); return } + if (policy.equalsIgnoreCase("Strict") || trackable.isEmpty) { logger.info(ctx, s"viewer.lp: optionality computed(strict) optional=[] | user=$userId root=$rootId"); writeOptionalNodes(userId, rootId, batchId, Set.empty, ctx); return } val diagnostic = trackable.head val hasDiagnostic = isAssessmentCourse(diagnostic, ctx) if (hasDiagnostic && !courseComplete(diagnostic)) return // wait for the diagnostic @@ -249,7 +258,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { }} HttpClientUtil.post(base + "/v1/course/enroll", body, headers, ctx) } - logger.info(ctx, s"ViewerAggregatorActor: system-enrol requested course=$courseId ctx=$batchId user=$userId mode=${ProjectUtil.getConfigValue("deployment_mode")}") + logger.info(ctx, s"viewer.lp: enrol dispatched | user=$userId course=$courseId batch=$batchId mode=${ProjectUtil.getConfigValue("deployment_mode")}") } private def writeOptionalNodes(userId: String, rootId: String, batchId: String, optional: Set[String], ctx: RequestContext): Unit = { @@ -334,7 +343,7 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { }} cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) if (status == 2 && currentStatus != 2) { - logger.info(ctx, s"ViewerAggregatorActor: node completed userId=$userId courseId=$nodeId; issuing cert") + logger.info(ctx, s"viewer.rollup: node completed -> cert | user=$userId course=$nodeId batch=$nodeCtx") certificateUtil.publishCertificateIssueEvent(userId, nodeId, nodeCtx, ctx) } } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala index 16942a04..bcfc8b65 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerSummaryActor.scala @@ -52,7 +52,7 @@ class ViewerSummaryActor extends BaseEnrolmentActor { if (StringUtils.isNotBlank(courseId)) enrolFilters.put("courseid", courseId) if (StringUtils.isNotBlank(batchId)) enrolFilters.put("batchid", batchId) val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, enrolFilters, ctx) - + logger.info(ctx, s"summary: read | user=$userId course=$courseId rows=${enrolments.size}") val response = new Response response.put(JsonKey.RESPONSE, enrolments) sender().tell(response, self) @@ -66,6 +66,7 @@ class ViewerSummaryActor extends BaseEnrolmentActor { val filters = new util.HashMap[String, AnyRef]() filters.put("userid", userId) val enrolments = getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters, ctx) + logger.info(ctx, s"summary: list | user=$userId rows=${enrolments.size}") val response = new Response response.put(JsonKey.RESPONSE, enrolments) sender().tell(response, self) @@ -86,6 +87,7 @@ class ViewerSummaryActor extends BaseEnrolmentActor { response.put("format", format) if (format == "csv") response.put("url", uploadSummaryCsv(userId, toCsv(enrolments))) else response.put(JsonKey.RESPONSE, enrolments) + logger.info(ctx, s"summary: download | user=$userId format=$format rows=${enrolments.size}") sender().tell(response, self) } @@ -137,6 +139,7 @@ class ViewerSummaryActor extends BaseEnrolmentActor { } else { deleteEnrolment(userId, courseId, batchId, ctx) } + logger.info(ctx, s"summary: delete | user=$userId course=${Option(courseId).getOrElse("ALL")}") sender().tell(successResponse(), self) } From 3118699b373598f767e7f1467466bfa11cbc1c0e Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Mon, 10 Aug 2026 13:38:26 +0530 Subject: [PATCH 26/30] feat(viewer): Derive course ancestors from leaf nodes ViewerAggregatorActor: course ancestors are not published, so derive a course's ancestor chain from one of its leaf nodes (use getLeafNodes(...).headOption -> getAncestors; fallback to empty list). Updated ProgressionPolicySpec with two tests to cover leaf ancestor chains and courses directly under root to ensure levelOf picks the correct level in both cases. --- .../sunbird/viewer/actor/ViewerAggregatorActor.scala | 7 ++++++- .../sunbird/viewer/util/ProgressionPolicySpec.scala | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index 59defa41..ee8c89ae 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -151,7 +151,12 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { ensureOptionalityComputed(userId, rootId, batchId, trackable, courseComplete, ctx) val optional = readOptionalNodes(userId, rootId, batchId, ctx).toSet - val ancestorsOf = (n: String) => hierarchyRelationsUtil.getAncestors(rootId, n, ctx) + // Course ancestors aren't published (only leaf ancestors are), so derive a course's chain from one of + // its leaves: leaf ancestors = [..course, level, root] (root LAST) -> levelOf = last non-root = the level. + val ancestorsOf = (course: String) => + hierarchyRelationsUtil.getLeafNodes(rootId, course, ctx).headOption + .map(leaf => hierarchyRelationsUtil.getAncestors(rootId, leaf, ctx)) + .getOrElse(List.empty[String]) val levels = ProgressionPolicy.orderedLevels(trackable, ancestorsOf, rootId) // Level complete = all its required (non-optional) courses complete (empty required set = complete, §5). diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala index 195d8646..7705a0d6 100644 --- a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala @@ -23,6 +23,18 @@ class ProgressionPolicySpec extends AnyFlatSpec with Matchers { ProgressionPolicy.levelOf("CRS-X", ancestorsOf, "do_lp") shouldBe Some("L2") } + // Hybrid: levels are derived from a LEAF's ancestors (root LAST), which also contain the unit + course. + // levelOf must still pick the level = last non-root, regardless of the extra leading nodes. + it should "pick the level from a full leaf ancestor chain [unit, course, level, root]" in { + val leafChain: String => List[String] = _ => List("U1", "CRS-B", "L2", "do_lp") + ProgressionPolicy.levelOf("CRS-B", leafChain, "do_lp") shouldBe Some("L2") + } + + it should "treat a course directly under root (no level wrapper) as its own level" in { + val flatChain: String => List[String] = _ => List("U1", "CRS-Z", "do_lp") + ProgressionPolicy.levelOf("CRS-Z", flatChain, "do_lp") shouldBe Some("CRS-Z") + } + "coursesOfLevel" should "group the level's courses in trackablenodes order" in { ProgressionPolicy.coursesOfLevel("L2", order, ancestorsOf, "do_lp") shouldBe List("CRS-B", "CRS-C") } From f4751ec3aea3ee0b801a5bc2d8c8222490f943a1 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 11 Aug 2026 15:22:01 +0530 Subject: [PATCH 27/30] feat(lp): wire Strict/Adaptive/PriorLearning policy engine - LpPolicyUtil: cached /v3/search per LP + long-TTL framework->last-category-code cache; policy/courseMeta/isAssessment/questionSets/skillsOfQuestions (reads , not se_Ids) - ViewerAggregatorActor: delegate policy/assessment/skill stubs; pre-assessment = first-level assessment course; Adaptive w/o pre-assessment warns+exits; PriorLearning waives prior-completed (any-batch status=2); skillsFromAssessment from assessment_aggregator best attempt (fully-correct). Engine & creditSkills unchanged - ProgressionPolicy.computeOptionalNodes: +priorCompleted - cache TTL config keys --- .../resources/externalresource.properties | 2 + .../sunbird/activity/util/LpPolicyUtil.scala | 142 ++++++++++++++++++ .../viewer/actor/ViewerAggregatorActor.scala | 82 ++++++---- .../viewer/util/ProgressionPolicy.scala | 9 +- .../viewer/util/LpPolicyUtilLogicSpec.scala | 22 +++ .../viewer/util/LpPolicyUtilParseSpec.scala | 40 +++++ .../viewer/util/ProgressionPolicySpec.scala | 22 +++ 7 files changed, 288 insertions(+), 31 deletions(-) create mode 100644 modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala create mode 100644 modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilLogicSpec.scala create mode 100644 modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilParseSpec.scala diff --git a/core/sunbird-platform-common/src/main/resources/externalresource.properties b/core/sunbird-platform-common/src/main/resources/externalresource.properties index 975b627b..c184eb03 100644 --- a/core/sunbird-platform-common/src/main/resources/externalresource.properties +++ b/core/sunbird-platform-common/src/main/resources/externalresource.properties @@ -182,6 +182,8 @@ sunbird_username_num_digits=4 sunbird_user_bulk_upload_size=1001 bulk_upload_org_data_size=300 sunbird_framework_read_api=/v1/framework/read +lp_meta_cache_ttl=3600 +framework_category_cache_ttl=86400 sunbird.channel.create.api.url=/channel/v3/create sunbird.channel.update.api.url=/channel/v3/update frameworkvalidation=false diff --git a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala new file mode 100644 index 00000000..889c4728 --- /dev/null +++ b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala @@ -0,0 +1,142 @@ +package org.sunbird.activity.util + +import com.fasterxml.jackson.databind.ObjectMapper +import org.sunbird.common.ProjectUtil +import org.sunbird.http.HttpUtil +import org.sunbird.request.RequestContext + +import java.util +import scala.collection.JavaConverters._ + +case class NodeMeta(primaryCategory: String, skills: Set[String], childNodes: List[String]) +case class LpMeta(policy: String, framework: String, nodes: Map[String, NodeMeta]) + +/** + * LP content metadata via one cached /v3/search per LP + a long-TTL framework->last-category-code cache. + * Pure parse/derive functions on the companion are I/O-free and unit-tested; the class wraps them with + * search + framework-read + caching. Reads the framework last-category code field (e.g. "skill"), never se_*Ids. + */ +object LpPolicyUtil { + private val mapper = new ObjectMapper() + + private def result(json: String): util.Map[String, AnyRef] = + mapper.readValue(json, classOf[util.Map[String, AnyRef]]) + .getOrDefault("result", new util.HashMap[String, AnyRef]()).asInstanceOf[util.Map[String, AnyRef]] + + // every array-valued key under result (Content / Question / content), flattened + private def rows(res: util.Map[String, AnyRef]): List[util.Map[String, AnyRef]] = + res.asScala.values.collect { case l: util.List[_] => + l.asInstanceOf[util.List[util.Map[String, AnyRef]]].asScala }.flatten.toList + + private def strs(v: AnyRef): Set[String] = v match { + case l: util.List[_] => l.asScala.map(_.toString).toSet + case s: String if s.nonEmpty => Set(s) + case _ => Set.empty + } + + def parseFrameworkCategoryCode(json: String): Option[String] = { + val fw = result(json).getOrDefault("framework", new util.HashMap[String, AnyRef]()).asInstanceOf[util.Map[String, AnyRef]] + val cats = Option(fw.get("categories")).collect { case l: util.List[_] => + l.asInstanceOf[util.List[util.Map[String, AnyRef]]].asScala.toList }.getOrElse(Nil) + if (cats.isEmpty) None + else Some(cats.maxBy(c => Option(c.get("index")).map(_.asInstanceOf[Number].doubleValue()).getOrElse(0.0)) + .get("code").toString) + } + + def parseLpNodes(json: String, categoryCode: String): Map[String, NodeMeta] = + rows(result(json)).flatMap { r => + Option(r.get("identifier")).map(_.toString).map { id => + id -> NodeMeta( + Option(r.get("primaryCategory")).map(_.toString).getOrElse(""), + Option(if (categoryCode.isEmpty) null else r.get(categoryCode)).map(strs).getOrElse(Set.empty), + Option(r.get("childNodes")).collect { case l: util.List[_] => l.asScala.map(_.toString).toList }.getOrElse(Nil)) + } + }.toMap + + def parseField(json: String, id: String, field: String): Option[String] = + rows(result(json)).find(r => Option(r.get("identifier")).map(_.toString).contains(id)) + .flatMap(r => Option(r.get(field)).map(_.toString)) + + // ---- pure derived accessors ---- + def isAssessment(courseId: String, nodes: Map[String, NodeMeta]): Boolean = { + val self = nodes.get(courseId) + self.exists(_.primaryCategory == "Practice Question Set") || + self.toList.flatMap(_.childNodes).exists(id => nodes.get(id).exists(_.primaryCategory == "Practice Question Set")) + } + def questionSets(courseId: String, nodes: Map[String, NodeMeta]): List[String] = + nodes.get(courseId).toList.flatMap(_.childNodes).filter(id => nodes.get(id).exists(_.primaryCategory == "Practice Question Set")) + + // ---- long-TTL caches (framework changes are rare; LP structure changes only on republish) ---- + private val metaTtl: Long = + Option(ProjectUtil.getConfigValue("lp_meta_cache_ttl")).filter(_.trim.nonEmpty).map(_.trim.toLong).getOrElse(3600L) * 1000L + private val codeTtl: Long = + Option(ProjectUtil.getConfigValue("framework_category_cache_ttl")).filter(_.trim.nonEmpty).map(_.trim.toLong).getOrElse(86400L) * 1000L + private val metaCache = new java.util.concurrent.ConcurrentHashMap[String, (Long, LpMeta)]() + private val codeCache = new java.util.concurrent.ConcurrentHashMap[String, (Long, String)]() + private def cachedMeta(k: String)(load: => LpMeta): LpMeta = { + val now = System.currentTimeMillis(); val h = metaCache.get(k) + if (h != null && h._1 > now) h._2 else { val v = load; if (v.nodes.nonEmpty) metaCache.put(k, (now + metaTtl, v)); v } + } + private def cachedCode(k: String)(load: => Option[String]): Option[String] = { + val now = System.currentTimeMillis(); val h = codeCache.get(k) + if (h != null && h._1 > now) Some(h._2) else { val v = load; v.foreach(c => codeCache.put(k, (now + codeTtl, c))); v } + } + + def apply(): LpPolicyUtil = new LpPolicyUtil() +} + +class LpPolicyUtil { + import LpPolicyUtil._ + + private def searchUrl: String = ProjectUtil.getConfigValue("service_search_base_path") + "/v3/search" + + private def post(body: String): String = { + val r = HttpUtil.doPostRequest(searchUrl, body, new util.HashMap[String, String]()) + if (r != null && r.getStatusCode == 200) r.getBody else "{}" + } + + private def searchByIds(ids: List[String], fields: List[String]): String = { + if (ids.isEmpty || fields.isEmpty) return "{}" + val idArr = ids.map(i => "\"" + i + "\"").mkString(",") + val fldArr = fields.map(f => "\"" + f + "\"").mkString(",") + post(s"""{"request":{"filters":{"status":["Live"],"identifier":[$idArr]},"fields":[$fldArr]}}""") + } + + private def frameworkCategoryCode(frameworkId: String): Option[String] = + if (frameworkId == null || frameworkId.isEmpty) None + else cachedCode(frameworkId) { + val base = ProjectUtil.getConfigValue("content_service_base_url") + val api = Option(ProjectUtil.getConfigValue("sunbird_framework_read_api")).filter(_.nonEmpty).getOrElse("/v1/framework/read") + val body = HttpUtil.sendGetRequest(base + api + "/" + frameworkId, new util.HashMap[String, String]()) + parseFrameworkCategoryCode(if (body == null) "{}" else body) + } + + def lpMeta(rootId: String, ctx: RequestContext): LpMeta = cachedMeta(rootId) { + val rootJson = searchByIds(List(rootId), List("policy", "framework", "childNodes")) + val policy = parseField(rootJson, rootId, "policy").getOrElse("Strict") + val framework = parseField(rootJson, rootId, "framework").getOrElse("") + val childNodes = parseLpNodes(rootJson, "").get(rootId).map(_.childNodes).getOrElse(Nil) + val categoryCode = frameworkCategoryCode(framework).getOrElse("") + val ids = (rootId :: childNodes).distinct + val fields = if (categoryCode.isEmpty) List("primaryCategory", "childNodes") else List(categoryCode, "primaryCategory", "childNodes") + LpMeta(policy, framework, parseLpNodes(searchByIds(ids, fields), categoryCode)) + } + + def policyOf(meta: LpMeta): String = meta.policy match { + case p if p != null && p.equalsIgnoreCase("Adaptive") => "Adaptive" + case p if p != null && p.equalsIgnoreCase("PriorLearning") => "PriorLearning" + case _ => "Strict" + } + def isAssessmentCourse(courseId: String, meta: LpMeta): Boolean = isAssessment(courseId, meta.nodes) + def questionSetsOf(courseId: String, meta: LpMeta): List[String] = questionSets(courseId, meta.nodes) + def courseMeta(courseIds: List[String], meta: LpMeta): Map[String, (Set[String], Boolean)] = + courseIds.map(c => c -> (meta.nodes.get(c).map(_.skills).getOrElse(Set.empty), isAssessment(c, meta.nodes))).toMap + + // terms for a set of question identifiers (one /v3/search) + def skillsOfQuestions(questionIds: List[String], meta: LpMeta): Set[String] = { + if (questionIds.isEmpty) return Set.empty + val code = frameworkCategoryCode(meta.framework).getOrElse("") + if (code.isEmpty) return Set.empty + parseLpNodes(searchByIds(questionIds, List(code)), code).values.flatMap(_.skills).toSet + } +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index ee8c89ae..22ec3956 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -37,6 +37,8 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { private var hierarchyRelationsUtil: HierarchyRelationsUtil = HierarchyRelationsUtil(cassandraOperation) private var certificateUtil: CertificateUtil = CertificateUtil() private val activityAggUtil = new ActivityAggregateUtil() + private val lpPolicyUtil: org.sunbird.activity.util.LpPolicyUtil = org.sunbird.activity.util.LpPolicyUtil() + private val assessmentService = new org.sunbird.assessment.service.CassandraService(Some(cassandraOperation)) private val enrolmentDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_COURSE_DB) private val activityAggDBInfo = Util.dbInfoMap.get(JsonKey.GROUP_ACTIVITY_DB) @@ -149,14 +151,15 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { val childBatchOf = (c: String) => batchId + ":" + c val courseComplete = (c: String) => status.get((c, childBatchOf(c))).contains(2) - ensureOptionalityComputed(userId, rootId, batchId, trackable, courseComplete, ctx) - val optional = readOptionalNodes(userId, rootId, batchId, ctx).toSet // Course ancestors aren't published (only leaf ancestors are), so derive a course's chain from one of // its leaves: leaf ancestors = [..course, level, root] (root LAST) -> levelOf = last non-root = the level. val ancestorsOf = (course: String) => hierarchyRelationsUtil.getLeafNodes(rootId, course, ctx).headOption .map(leaf => hierarchyRelationsUtil.getAncestors(rootId, leaf, ctx)) .getOrElse(List.empty[String]) + + ensureOptionalityComputed(userId, rootId, batchId, trackable, ancestorsOf, status, ctx) + val optional = readOptionalNodes(userId, rootId, batchId, ctx).toSet val levels = ProgressionPolicy.orderedLevels(trackable, ancestorsOf, rootId) // Level complete = all its required (non-optional) courses complete (empty required set = complete, §5). @@ -180,39 +183,62 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // LP completion = every level complete -> credit durable skills (once; creditSkills no-ops if nothing new). if (levels.nonEmpty && levels.forall(levelComplete)) { logger.info(ctx, s"viewer.lp: complete | user=$userId root=$rootId") - creditSkills(userId, rootId, trackable, ctx) + creditSkills(userId, rootId, batchId, trackable, ctx) } } private def ensureOptionalityComputed(userId: String, rootId: String, batchId: String, trackable: List[String], - courseComplete: String => Boolean, ctx: RequestContext): Unit = { + ancestorsOf: String => List[String], + status: Map[(String, String), Int], ctx: RequestContext): Unit = { // Compute once (§Step 4); optionalityComputed remembers empty results without a DB column. if (optionalityComputed(userId, rootId, batchId, ctx)) return - val policy = policyOf(rootId, ctx) - if (policy.equalsIgnoreCase("Strict") || trackable.isEmpty) { logger.info(ctx, s"viewer.lp: optionality computed(strict) optional=[] | user=$userId root=$rootId"); writeOptionalNodes(userId, rootId, batchId, Set.empty, ctx); return } - val diagnostic = trackable.head - val hasDiagnostic = isAssessmentCourse(diagnostic, ctx) - if (hasDiagnostic && !courseComplete(diagnostic)) return // wait for the diagnostic - val prior = if (policy.equalsIgnoreCase("PriorLearning")) readUserSkills(userId, ctx) else Set.empty[String] - val fromDiag = if (hasDiagnostic) skillsFromAssessment(userId, rootId, diagnostic, ctx) else Set.empty[String] - val achieved = prior ++ fromDiag - val meta = courseMeta(trackable, ctx) - val assessmentCourses = meta.collect { case (c, (_, true)) => c }.toSet - val skillsByCourse = meta.map { case (c, (s, _)) => c -> s } + val meta = lpPolicyUtil.lpMeta(rootId, ctx) + val policy = lpPolicyUtil.policyOf(meta) + if (policy.equalsIgnoreCase("Strict") || trackable.isEmpty) { + logger.info(ctx, s"viewer.lp: optionality computed(strict) optional=[] | user=$userId root=$rootId") + writeOptionalNodes(userId, rootId, batchId, Set.empty, ctx); return + } + // pre-assessment = an assessment course in the FIRST level (seeds achieved skills before waivers). + val levels = ProgressionPolicy.orderedLevels(trackable, ancestorsOf, rootId) + val preAssessment = ProgressionPolicy.coursesOfLevel(levels.headOption.getOrElse(""), trackable, ancestorsOf, rootId) + .find(c => lpPolicyUtil.isAssessmentCourse(c, meta)) + if (policy.equalsIgnoreCase("Adaptive") && preAssessment.isEmpty) { + logger.warn(ctx, s"viewer.lp: Adaptive LP has no pre-assessment; exiting (misconfigured) | user=$userId root=$rootId", null) + return + } + val childBatchOf = (c: String) => batchId + ":" + c + val courseComplete = (c: String) => status.get((c, childBatchOf(c))).contains(2) + if (preAssessment.exists(pa => !courseComplete(pa))) return // wait for the pre-assessment + val achieved = preAssessment.map(pa => skillsFromAssessment(userId, rootId, pa, batchId, ctx)).getOrElse(Set.empty) + // PriorLearning: a content course already completed under ANY batch is optional directly. + val priorCompleted = + if (policy.equalsIgnoreCase("PriorLearning")) trackable.filter(c => status.exists { case ((cc, _), st) => cc == c && st == 2 }).toSet + else Set.empty[String] + val cMeta = lpPolicyUtil.courseMeta(trackable, meta) + val assessmentCourses = cMeta.collect { case (c, (_, true)) => c }.toSet + val skillsByCourse = cMeta.map { case (c, (s, _)) => c -> s } + logger.info(ctx, s"viewer.lp: optionality computing($policy) preAssess=${preAssessment.getOrElse("-")} achieved=${achieved.size} priorDone=${priorCompleted.size} | user=$userId root=$rootId") writeOptionalNodes(userId, rootId, batchId, - ProgressionPolicy.computeOptionalNodes(policy, trackable, skillsByCourse, assessmentCourses, achieved), ctx) + ProgressionPolicy.computeOptionalNodes(policy, trackable, skillsByCourse, assessmentCourses, achieved, priorCompleted), ctx) } - // VERIFY-ON-DEPLOY: policy source. Read the LP's policy from collection/batch metadata; absent => Strict. - private def policyOf(rootId: String, ctx: RequestContext): String = "Strict" - // VERIFY-ON-DEPLOY: assessment detection. Needs a Practice-Question-Set child via /v3/search or content_hierarchy. - private def isAssessmentCourse(courseId: String, ctx: RequestContext): Boolean = false - // VERIFY-ON-DEPLOY: per-course skills + assessment flag via /v3/search. Skills = framework last-category - // terms (se_Ids), not an se_skills field (see design §6). - private def courseMeta(trackable: List[String], ctx: RequestContext): Map[String, (Set[String], Boolean)] = - trackable.map(c => c -> (Set.empty[String], isAssessmentCourse(c, ctx))).toMap - // VERIFY-ON-DEPLOY: achieved = assessment_aggregator best attempts × question skill ids (all correct); ids = framework last-category terms (design §6). - private def skillsFromAssessment(userId: String, rootId: String, courseId: String, ctx: RequestContext): Set[String] = Set.empty + private def isAssessmentCourse(rootId: String, courseId: String, ctx: RequestContext): Boolean = + lpPolicyUtil.isAssessmentCourse(courseId, lpPolicyUtil.lpMeta(rootId, ctx)) + + // terms (framework last-category code, e.g. "skill") of the questions the learner got fully + // correct in the pre-assessment's question set(s). courseId = the pre-assessment course; its child batch + // is rootBatch:courseId. Never se_Ids. + private def skillsFromAssessment(userId: String, rootId: String, courseId: String, batchId: String, ctx: RequestContext): Set[String] = { + val meta = lpPolicyUtil.lpMeta(rootId, ctx) + val childBatch = batchId + ":" + courseId + // best attempt (highest total) per question set; keep only fully-correct questions. + val correct = lpPolicyUtil.questionSetsOf(courseId, meta).flatMap { qs => + val attempts = assessmentService.getUserAssessments(userId, courseId, childBatch, qs, ctx) + if (attempts.isEmpty) Nil + else attempts.maxBy(_.totalScore).questions.collect { case q if q.maxScore > 0 && q.score == q.maxScore => q.questionId } + }.distinct + lpPolicyUtil.skillsOfQuestions(correct, meta) + } /** * All of this user's enrolments as (courseId, batchId) -> status, in one read. Replaces the LP's former @@ -289,8 +315,8 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { } // Durable skill credit — ONCE at LP completion. Read-union-upsert (portable; no set-append needed). - private def creditSkills(userId: String, rootId: String, trackable: List[String], ctx: RequestContext): Unit = { - val earned = trackable.filter(c => isAssessmentCourse(c, ctx)).flatMap(c => skillsFromAssessment(userId, rootId, c, ctx)).toSet + private def creditSkills(userId: String, rootId: String, batchId: String, trackable: List[String], ctx: RequestContext): Unit = { + val earned = trackable.filter(c => isAssessmentCourse(rootId, c, ctx)).flatMap(c => skillsFromAssessment(userId, rootId, c, batchId, ctx)).toSet if (earned.isEmpty) return // no-op until se_skills is wired (VERIFY-ON-DEPLOY) val existing = readUserSkills(userId, ctx) val merged = existing ++ earned diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala index 1756aa49..40cc576d 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala @@ -31,11 +31,14 @@ object ProgressionPolicy { def computeOptionalNodes(policy: String, courses: List[String], skillsByCourse: Map[String, Set[String]], assessmentCourses: Set[String], - skillsAchieved: Set[String]): Set[String] = { + skillsAchieved: Set[String], + priorCompleted: Set[String] = Set.empty): Set[String] = { if ("Strict".equalsIgnoreCase(policy)) Set.empty else courses.filter { c => - val skills = skillsByCourse.getOrElse(c, Set.empty) - !assessmentCourses.contains(c) && skills.nonEmpty && skills.subsetOf(skillsAchieved) + !assessmentCourses.contains(c) && { + val skills = skillsByCourse.getOrElse(c, Set.empty) + priorCompleted.contains(c) || (skills.nonEmpty && skills.subsetOf(skillsAchieved)) + } }.toSet } } diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilLogicSpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilLogicSpec.scala new file mode 100644 index 00000000..05313722 --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilLogicSpec.scala @@ -0,0 +1,22 @@ +package org.sunbird.viewer.util + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.activity.util.{LpPolicyUtil, NodeMeta} + +class LpPolicyUtilLogicSpec extends AnyFlatSpec with Matchers { + private val nodes = Map( + "crsA" -> NodeMeta("Course", Set("Python Programming"), List("qsA")), + "qsA" -> NodeMeta("Practice Question Set", Set.empty, Nil), + "crsB" -> NodeMeta("Course", Set("JavaScript"), Nil)) + + "isAssessment" should "be true when a child is a Practice Question Set" in { + LpPolicyUtil.isAssessment("crsA", nodes) shouldBe true + LpPolicyUtil.isAssessment("crsB", nodes) shouldBe false + } + + "questionSets" should "list the Practice-Question-Set children" in { + LpPolicyUtil.questionSets("crsA", nodes) shouldBe List("qsA") + LpPolicyUtil.questionSets("crsB", nodes) shouldBe Nil + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilParseSpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilParseSpec.scala new file mode 100644 index 00000000..a894ecb4 --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/LpPolicyUtilParseSpec.scala @@ -0,0 +1,40 @@ +package org.sunbird.viewer.util + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.activity.util.{LpPolicyUtil, NodeMeta} + +class LpPolicyUtilParseSpec extends AnyFlatSpec with Matchers { + + "parseFrameworkCategoryCode" should "return the highest-index category code" in { + val json = """{"result":{"framework":{"identifier":"USF","categories":[ + {"code":"board","index":1},{"code":"subject","index":2},{"code":"skill","index":3}]}}}""" + LpPolicyUtil.parseFrameworkCategoryCode(json) shouldBe Some("skill") + } + + it should "return None when there are no categories" in { + LpPolicyUtil.parseFrameworkCategoryCode("""{"result":{"framework":{"categories":[]}}}""") shouldBe None + } + + "parseLpNodes" should "map each node to its primaryCategory, terms and childNodes" in { + val json = """{"result":{"count":2,"Content":[ + {"identifier":"c1","primaryCategory":"Course","skill":["Python Programming"],"childNodes":["q1"]}, + {"identifier":"c2","primaryCategory":"Practice Question Set","childNodes":[]}]}}""" + val nodes = LpPolicyUtil.parseLpNodes(json, "skill") + nodes("c1") shouldBe NodeMeta("Course", Set("Python Programming"), List("q1")) + nodes("c2").primaryCategory shouldBe "Practice Question Set" + nodes("c2").skills shouldBe empty + } + + it should "read whichever objectType array key is present (Question), not just content" in { + val json = """{"result":{"count":1,"Question":[ + {"identifier":"q1","primaryCategory":"Practice Question Set","skill":["JavaScript"]}]}}""" + LpPolicyUtil.parseLpNodes(json, "skill")("q1").skills shouldBe Set("JavaScript") + } + + "parseField" should "read a scalar field for an identifier" in { + val json = """{"result":{"content":[{"identifier":"root","policy":"PriorLearning","framework":"USF"}]}}""" + LpPolicyUtil.parseField(json, "root", "policy") shouldBe Some("PriorLearning") + LpPolicyUtil.parseField(json, "root", "framework") shouldBe Some("USF") + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala index 7705a0d6..50752059 100644 --- a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala @@ -57,4 +57,26 @@ class ProgressionPolicySpec extends AnyFlatSpec with Matchers { ProgressionPolicy.computeOptionalNodes("Strict", List("CRS-B"), Map("CRS-B" -> Set("s1")), Set.empty, Set("s1")) shouldBe empty } + + it should "waive a prior-completed course regardless of skills (PriorLearning)" in { + val opt = ProgressionPolicy.computeOptionalNodes( + policy = "PriorLearning", + courses = List("CRS-B", "CRS-C"), + skillsByCourse = Map("CRS-B" -> Set.empty, "CRS-C" -> Set.empty), + assessmentCourses = Set.empty, + skillsAchieved = Set.empty, + priorCompleted = Set("CRS-B")) + opt shouldBe Set("CRS-B") + } + + it should "never waive an assessment course even if prior-completed" in { + val opt = ProgressionPolicy.computeOptionalNodes( + policy = "PriorLearning", + courses = List("CRS-B"), + skillsByCourse = Map("CRS-B" -> Set.empty), + assessmentCourses = Set("CRS-B"), + skillsAchieved = Set.empty, + priorCompleted = Set("CRS-B")) + opt shouldBe empty + } } From d2bf8ce4001ac1151a8fe104eb647f4bcce6c74b Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Tue, 11 Aug 2026 17:28:13 +0530 Subject: [PATCH 28/30] =?UTF-8?q?refactor(lp):=20viewer=20quality=20pass?= =?UTF-8?q?=20=E2=80=94=20dedup,=20OOP=20extraction,=20batch-exists=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ViewerAggregatorActor: extract LP progression to injectable LpProgressionEngine (SRP/DIP) + EnrolDispatcher trait (OCP, monolith/HTTP); actor 445->288 lines - A: precompute course->level map once; completedCourses Set; LpMeta.categoryCode (drop framework re-resolve) - LpPolicyUtil.searchByIds: mapper-built JSON (escaping) - CourseBatchManagementActor.batchExists: reuse readById, narrow catch so a transient DB error isn't misread as absent (no duplicate child batch) - Adaptive LP with no pre-assessment now halts (opens nothing) instead of opening the first course - Tests: LpProgressionEngineSpec (Strict/waiver/Adaptive-halt) + levelByCourse --- .../sunbird/activity/util/LpPolicyUtil.scala | 23 +-- .../CourseBatchManagementActor.java | 7 +- .../viewer/actor/ViewerAggregatorActor.scala | 178 +----------------- .../viewer/engine/EnrolDispatcher.scala | 53 ++++++ .../viewer/engine/LpProgressionEngine.scala | 166 ++++++++++++++++ .../viewer/util/ProgressionPolicy.scala | 10 + .../engine/LpProgressionEngineSpec.scala | 91 +++++++++ .../viewer/util/ProgressionPolicySpec.scala | 9 + 8 files changed, 350 insertions(+), 187 deletions(-) create mode 100644 modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/EnrolDispatcher.scala create mode 100644 modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala create mode 100644 modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala diff --git a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala index 889c4728..5cc72158 100644 --- a/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala +++ b/modules/lms/activity-aggregator/src/main/scala/org/sunbird/activity/util/LpPolicyUtil.scala @@ -9,13 +9,10 @@ import java.util import scala.collection.JavaConverters._ case class NodeMeta(primaryCategory: String, skills: Set[String], childNodes: List[String]) -case class LpMeta(policy: String, framework: String, nodes: Map[String, NodeMeta]) +case class LpMeta(policy: String, framework: String, categoryCode: String, nodes: Map[String, NodeMeta]) -/** - * LP content metadata via one cached /v3/search per LP + a long-TTL framework->last-category-code cache. - * Pure parse/derive functions on the companion are I/O-free and unit-tested; the class wraps them with - * search + framework-read + caching. Reads the framework last-category code field (e.g. "skill"), never se_*Ids. - */ +/** LP content metadata via a cached /v3/search + long-TTL framework->last-category-code cache. Reads the + * category code field (e.g. "skill"), never se_*Ids. Pure parsers on the companion; class does I/O + caching. */ object LpPolicyUtil { private val mapper = new ObjectMapper() @@ -97,9 +94,9 @@ class LpPolicyUtil { private def searchByIds(ids: List[String], fields: List[String]): String = { if (ids.isEmpty || fields.isEmpty) return "{}" - val idArr = ids.map(i => "\"" + i + "\"").mkString(",") - val fldArr = fields.map(f => "\"" + f + "\"").mkString(",") - post(s"""{"request":{"filters":{"status":["Live"],"identifier":[$idArr]},"fields":[$fldArr]}}""") + val filters = new util.HashMap[String, AnyRef]() {{ put("status", util.Arrays.asList("Live")); put("identifier", ids.asJava) }} + val request = new util.HashMap[String, AnyRef]() {{ put("filters", filters); put("fields", fields.asJava) }} + post(mapper.writeValueAsString(new util.HashMap[String, AnyRef]() {{ put("request", request) }})) } private def frameworkCategoryCode(frameworkId: String): Option[String] = @@ -119,7 +116,7 @@ class LpPolicyUtil { val categoryCode = frameworkCategoryCode(framework).getOrElse("") val ids = (rootId :: childNodes).distinct val fields = if (categoryCode.isEmpty) List("primaryCategory", "childNodes") else List(categoryCode, "primaryCategory", "childNodes") - LpMeta(policy, framework, parseLpNodes(searchByIds(ids, fields), categoryCode)) + LpMeta(policy, framework, categoryCode, parseLpNodes(searchByIds(ids, fields), categoryCode)) } def policyOf(meta: LpMeta): String = meta.policy match { @@ -134,9 +131,7 @@ class LpPolicyUtil { // terms for a set of question identifiers (one /v3/search) def skillsOfQuestions(questionIds: List[String], meta: LpMeta): Set[String] = { - if (questionIds.isEmpty) return Set.empty - val code = frameworkCategoryCode(meta.framework).getOrElse("") - if (code.isEmpty) return Set.empty - parseLpNodes(searchByIds(questionIds, List(code)), code).values.flatMap(_.skills).toSet + if (questionIds.isEmpty || meta.categoryCode.isEmpty) return Set.empty + parseLpNodes(searchByIds(questionIds, List(meta.categoryCode)), meta.categoryCode).values.flatMap(_.skills).toSet } } diff --git a/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java b/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java index 6cd22b53..3caabd9d 100644 --- a/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java +++ b/modules/lms/course-mw/course-actors/src/main/java/org/sunbird/learner/actors/coursebatch/CourseBatchManagementActor.java @@ -10,6 +10,7 @@ import org.sunbird.helper.ServiceFactory; import org.sunbird.common.ElasticSearchHelper; import org.sunbird.exception.ProjectCommonException; +import org.sunbird.response.ResponseCode; import org.sunbird.common.factory.EsClientFactory; import org.sunbird.common.inf.ElasticSearchService; import org.sunbird.response.Response; @@ -178,13 +179,15 @@ private void triggerChildBatchCreation(Request parent, String parentCourseId, St } } - /** Non-throwing existence check for a course_batch (readById throws when absent). */ + // Exists via readById; only invalidCourseBatchId means absent — any other PCE is rethrown so a transient + // error isn't misread as absent (which would create a duplicate batch). private boolean batchExists(String courseId, String batchId, RequestContext ctx) { try { courseBatchDao.readById(courseId, batchId, ctx); return true; } catch (ProjectCommonException e) { - return false; + if (ResponseCode.invalidCourseBatchId.getErrorCode().equals(e.getErrorCode())) return false; + throw e; } } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index 22ec3956..361ca914 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -5,13 +5,11 @@ import org.sunbird.activity.domain.{ContentStatus, UserContentConsumption, UserE import org.sunbird.activity.util.{ActivityAggregateUtil, CertificateUtil, HierarchyRelationsUtil} import org.sunbird.cassandra.CassandraOperation import org.sunbird.common.ProjectUtil -import org.sunbird.http.HttpClientUtil import org.sunbird.enrolments.BaseEnrolmentActor import org.sunbird.helper.ServiceFactory import org.sunbird.keys.JsonKey import org.sunbird.learner.util.Util import org.sunbird.request.{Request, RequestContext} -import org.sunbird.viewer.util.ProgressionPolicy import java.util import scala.collection.JavaConverters._ @@ -44,6 +42,11 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { private val activityAggDBInfo = Util.dbInfoMap.get(JsonKey.GROUP_ACTIVITY_DB) private val consumptionDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_CONTENT_DB) private val CONSUMPTION_TABLE = "user_content_consumption" + // LP progression extracted to a focused, injectable engine (SRP); transport behind a dispatcher (OCP). + // lazy so `context` is set by the time they initialize. + private lazy val enrolDispatcher: org.sunbird.viewer.engine.EnrolDispatcher = org.sunbird.viewer.engine.EnrolDispatcher(context) + private lazy val lpEngine = new org.sunbird.viewer.engine.LpProgressionEngine( + cassandraOperation, enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, lpPolicyUtil, assessmentService, enrolDispatcher) override def onReceive(request: Request): Unit = { request.getOperation match { @@ -134,110 +137,20 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { private def completedCountOf(a: UserEnrolmentAgg): Int = a.activityAgg.aggregates.getOrElse("completedCount", 0.0).toInt - // ─────────────────────────── LP progression (the engine) ─────────────────────────── - // Pure decisions come from ProgressionPolicy; this orchestrates reads/writes. Strict is fully - // functional. Adaptive/PriorLearning are wired but their skill inputs (policy source, se_skills, - // diagnostic assessment scores) are marked VERIFY-ON-DEPLOY — they default to "no skills" so the - // system compiles and behaves as Strict until those integrations are wired against the live env. - - private val USER_SKILLS_TABLE = "user_skills" - + // LP progression: read the status snapshot + derive ancestorsOf here, then delegate to LpProgressionEngine. private def advanceLp(userId: String, rootId: String, batchId: String, trackable: List[String], ctx: RequestContext): Unit = { // One read of this user's enrolments, reused for every completion/enrolment check below. Safe because // advanceLp runs AFTER writeAllNodeEnrolments has committed this pass's statuses, so the snapshot is // current; collapses the LP's former O(courses) single-row status reads into a single query. val status = enrolStatusSnapshot(userId, ctx) - val childBatchOf = (c: String) => batchId + ":" + c - val courseComplete = (c: String) => status.get((c, childBatchOf(c))).contains(2) - // Course ancestors aren't published (only leaf ancestors are), so derive a course's chain from one of // its leaves: leaf ancestors = [..course, level, root] (root LAST) -> levelOf = last non-root = the level. val ancestorsOf = (course: String) => hierarchyRelationsUtil.getLeafNodes(rootId, course, ctx).headOption .map(leaf => hierarchyRelationsUtil.getAncestors(rootId, leaf, ctx)) .getOrElse(List.empty[String]) - - ensureOptionalityComputed(userId, rootId, batchId, trackable, ancestorsOf, status, ctx) - val optional = readOptionalNodes(userId, rootId, batchId, ctx).toSet - val levels = ProgressionPolicy.orderedLevels(trackable, ancestorsOf, rootId) - - // Level complete = all its required (non-optional) courses complete (empty required set = complete, §5). - // Derived from persisted enrolment status only, so it's recompute-safe (force-sync repairs identically). - def levelComplete(level: String): Boolean = - ProgressionPolicy.coursesOfLevel(level, trackable, ancestorsOf, rootId).filterNot(optional.contains).forall(courseComplete) - - // Open first incomplete level: enrol all its optionals up front + the next single required course (§5). - levels.find(l => !levelComplete(l)).foreach { level => - val courses = ProgressionPolicy.coursesOfLevel(level, trackable, ancestorsOf, rootId) - val nextRequired = courses.filterNot(optional.contains).find(c => !courseComplete(c)) - val toOpen = courses.filter(optional.contains) ++ nextRequired.toList - logger.info(ctx, s"viewer.lp: level opened | user=$userId root=$rootId level=$level open=[${toOpen.mkString(",")}]") - toOpen.foreach { c => - val childBatch = childBatchOf(c) - if (!status.contains((c, childBatch))) internalEnrol(userId, c, childBatch, ctx) - else logger.info(ctx, s"viewer.lp: enrol skip(already) | user=$userId course=$c batch=$childBatch") - } - } - - // LP completion = every level complete -> credit durable skills (once; creditSkills no-ops if nothing new). - if (levels.nonEmpty && levels.forall(levelComplete)) { - logger.info(ctx, s"viewer.lp: complete | user=$userId root=$rootId") - creditSkills(userId, rootId, batchId, trackable, ctx) - } - } - - private def ensureOptionalityComputed(userId: String, rootId: String, batchId: String, trackable: List[String], - ancestorsOf: String => List[String], - status: Map[(String, String), Int], ctx: RequestContext): Unit = { - // Compute once (§Step 4); optionalityComputed remembers empty results without a DB column. - if (optionalityComputed(userId, rootId, batchId, ctx)) return - val meta = lpPolicyUtil.lpMeta(rootId, ctx) - val policy = lpPolicyUtil.policyOf(meta) - if (policy.equalsIgnoreCase("Strict") || trackable.isEmpty) { - logger.info(ctx, s"viewer.lp: optionality computed(strict) optional=[] | user=$userId root=$rootId") - writeOptionalNodes(userId, rootId, batchId, Set.empty, ctx); return - } - // pre-assessment = an assessment course in the FIRST level (seeds achieved skills before waivers). - val levels = ProgressionPolicy.orderedLevels(trackable, ancestorsOf, rootId) - val preAssessment = ProgressionPolicy.coursesOfLevel(levels.headOption.getOrElse(""), trackable, ancestorsOf, rootId) - .find(c => lpPolicyUtil.isAssessmentCourse(c, meta)) - if (policy.equalsIgnoreCase("Adaptive") && preAssessment.isEmpty) { - logger.warn(ctx, s"viewer.lp: Adaptive LP has no pre-assessment; exiting (misconfigured) | user=$userId root=$rootId", null) - return - } - val childBatchOf = (c: String) => batchId + ":" + c - val courseComplete = (c: String) => status.get((c, childBatchOf(c))).contains(2) - if (preAssessment.exists(pa => !courseComplete(pa))) return // wait for the pre-assessment - val achieved = preAssessment.map(pa => skillsFromAssessment(userId, rootId, pa, batchId, ctx)).getOrElse(Set.empty) - // PriorLearning: a content course already completed under ANY batch is optional directly. - val priorCompleted = - if (policy.equalsIgnoreCase("PriorLearning")) trackable.filter(c => status.exists { case ((cc, _), st) => cc == c && st == 2 }).toSet - else Set.empty[String] - val cMeta = lpPolicyUtil.courseMeta(trackable, meta) - val assessmentCourses = cMeta.collect { case (c, (_, true)) => c }.toSet - val skillsByCourse = cMeta.map { case (c, (s, _)) => c -> s } - logger.info(ctx, s"viewer.lp: optionality computing($policy) preAssess=${preAssessment.getOrElse("-")} achieved=${achieved.size} priorDone=${priorCompleted.size} | user=$userId root=$rootId") - writeOptionalNodes(userId, rootId, batchId, - ProgressionPolicy.computeOptionalNodes(policy, trackable, skillsByCourse, assessmentCourses, achieved, priorCompleted), ctx) - } - - private def isAssessmentCourse(rootId: String, courseId: String, ctx: RequestContext): Boolean = - lpPolicyUtil.isAssessmentCourse(courseId, lpPolicyUtil.lpMeta(rootId, ctx)) - - // terms (framework last-category code, e.g. "skill") of the questions the learner got fully - // correct in the pre-assessment's question set(s). courseId = the pre-assessment course; its child batch - // is rootBatch:courseId. Never se_Ids. - private def skillsFromAssessment(userId: String, rootId: String, courseId: String, batchId: String, ctx: RequestContext): Set[String] = { - val meta = lpPolicyUtil.lpMeta(rootId, ctx) - val childBatch = batchId + ":" + courseId - // best attempt (highest total) per question set; keep only fully-correct questions. - val correct = lpPolicyUtil.questionSetsOf(courseId, meta).flatMap { qs => - val attempts = assessmentService.getUserAssessments(userId, courseId, childBatch, qs, ctx) - if (attempts.isEmpty) Nil - else attempts.maxBy(_.totalScore).questions.collect { case q if q.maxScore > 0 && q.score == q.maxScore => q.questionId } - }.distinct - lpPolicyUtil.skillsOfQuestions(correct, meta) + lpEngine.advance(userId, rootId, batchId, trackable, status, ancestorsOf, ctx) } /** @@ -258,74 +171,6 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { }.toMap } - /** - * System-driven enrol via the standard `enrol` op. MONOLITH -> in-JVM message; DISTRIBUTED -> POST - * /v1/course/enroll. Re-enrol safe: advanceLp only enrols courses absent from the snapshot. - */ - private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) // default monolith - - private def internalEnrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = { - if (isMonolith) { - val req = new Request() - req.setRequestContext(ctx) - req.setRequestId("system") // enrol stores this as addedBy (via context REQUEST_ID) - req.setOperation("enrol") - req.put(JsonKey.USER_ID, userId); req.put(JsonKey.COURSE_ID, courseId); req.put(JsonKey.BATCH_ID, batchId) - // VERIFY-ON-DEPLOY: bound path of the enrolment actor in the monolith actor system. - val path = Option(ProjectUtil.getConfigValue("enrolment_actor_path")).filter(_.nonEmpty).getOrElse("/user/course-enrolment-actor") - // noSender: fire-and-forget; the enrol actor's success reply must NOT bounce back to this actor - // (it only handles "aggregate" Requests) — let the reply go to deadLetters. - context.actorSelection(path).tell(req, org.apache.pekko.actor.ActorRef.noSender) - } else { - // DISTRIBUTED: call the enrolment service over HTTP (full enrol op). - // VERIFY-ON-DEPLOY: use a system-enrol endpoint (not the public one that fans out/notifies) + forward auth token. - val base = Option(ProjectUtil.getConfigValue("enrolment_service_base_url")).filter(_.nonEmpty).getOrElse("http://lern-service:9000") - val body = s"""{"request":{"userId":"$userId","courseId":"$courseId","batchId":"$batchId"}}""" - val headers = new util.HashMap[String, String]() {{ - put("Content-Type", "application/json") - // System-driven enrol: authenticate with the configured system token (else 401 in distributed mode). - Option(ProjectUtil.getConfigValue("viewer_system_auth_token")).filter(_.nonEmpty) - .foreach(t => put("x-authenticated-user-token", t)) - }} - HttpClientUtil.post(base + "/v1/course/enroll", body, headers, ctx) - } - logger.info(ctx, s"viewer.lp: enrol dispatched | user=$userId course=$courseId batch=$batchId mode=${ProjectUtil.getConfigValue("deployment_mode")}") - } - - private def writeOptionalNodes(userId: String, rootId: String, batchId: String, optional: Set[String], ctx: RequestContext): Unit = { - val selectMap = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", rootId); put("batchid", batchId) }} - val updateMap = new util.HashMap[String, AnyRef]() {{ put("optional_nodes", optional.asJava) }} - cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) - ViewerAggregatorActor.markOptionalityComputed(userId, rootId, batchId) // remember empty results too (no DB column) - } - - /** Computed? Non-empty optional_nodes is self-evident; an empty result is remembered in an in-process memo. */ - private def optionalityComputed(userId: String, rootId: String, batchId: String, ctx: RequestContext): Boolean = - readOptionalNodes(userId, rootId, batchId, ctx).nonEmpty || - ViewerAggregatorActor.isOptionalityComputed(userId, rootId, batchId) - - private def readUserSkills(userId: String, ctx: RequestContext): Set[String] = { - val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} - val rows = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, USER_SKILLS_TABLE, - filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) - .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] - if (CollectionUtils.isNotEmpty(rows)) - Option(rows.get(0).get("skills")).map(_.asInstanceOf[util.Collection[String]].asScala.toSet).getOrElse(Set.empty) - else Set.empty - } - - // Durable skill credit — ONCE at LP completion. Read-union-upsert (portable; no set-append needed). - private def creditSkills(userId: String, rootId: String, batchId: String, trackable: List[String], ctx: RequestContext): Unit = { - val earned = trackable.filter(c => isAssessmentCourse(rootId, c, ctx)).flatMap(c => skillsFromAssessment(userId, rootId, c, batchId, ctx)).toSet - if (earned.isEmpty) return // no-op until se_skills is wired (VERIFY-ON-DEPLOY) - val existing = readUserSkills(userId, ctx) - val merged = existing ++ earned - if (merged.size == existing.size) return // nothing new — credit already granted (advanceLp calls this every completed pass) - val row = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("skills", merged.asJava) }} - cassandraOperation.insertRecord(enrolmentDBInfo.getKeySpace, USER_SKILLS_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) - logger.info(ctx, s"ViewerAggregatorActor: credited ${earned.size} skills to user=$userId for LP=$rootId") - } - private def writeActivityAggregates(aggs: List[UserEnrolmentAgg], ctx: RequestContext): Unit = { val aggQueries = aggs.map(a => activityAggUtil.createActivityAggUpdateMap(a.activityAgg)).asJava if (!aggQueries.isEmpty) @@ -433,13 +278,4 @@ object ViewerAggregatorActor { fresh.foreach { case (k, v) => merged.put(k, v) } merged } - - // JVM-wide memo of enrolments whose (empty) LP optionality is computed, so we don't recompute each pass. - // ponytail: unbounded set, entries live for the process lifetime; add a size cap / TTL only if it grows. - private val optionalityDone: java.util.Set[String] = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() - private def optKey(userId: String, rootId: String, batchId: String): String = s"$userId:$rootId:$batchId" - def markOptionalityComputed(userId: String, rootId: String, batchId: String): Unit = - optionalityDone.add(optKey(userId, rootId, batchId)) - def isOptionalityComputed(userId: String, rootId: String, batchId: String): Boolean = - optionalityDone.contains(optKey(userId, rootId, batchId)) } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/EnrolDispatcher.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/EnrolDispatcher.scala new file mode 100644 index 00000000..d29c715d --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/EnrolDispatcher.scala @@ -0,0 +1,53 @@ +package org.sunbird.viewer.engine + +import org.apache.pekko.actor.{ActorContext, ActorRef} +import org.sunbird.common.ProjectUtil +import org.sunbird.http.HttpClientUtil +import org.sunbird.keys.JsonKey +import org.sunbird.logging.LoggerUtil +import org.sunbird.request.{Request, RequestContext} + +import java.util + +/** System enrol via the standard `enrol` op; transport picked by deployment_mode (monolith tell vs HTTP POST). + * requestId="system" is stored as addedBy; re-enrol safety is the caller's. */ +trait EnrolDispatcher { + def enrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit +} + +object EnrolDispatcher { + private def isMonolith: Boolean = !"distributed".equalsIgnoreCase(ProjectUtil.getConfigValue("deployment_mode")) // default monolith + def apply(context: ActorContext): EnrolDispatcher = + if (isMonolith) new MonolithEnrolDispatcher(context) else new HttpEnrolDispatcher() +} + +/** In-JVM: tell the enrolment actor. noSender so its success reply goes to deadLetters, not the caller. */ +class MonolithEnrolDispatcher(context: ActorContext) extends EnrolDispatcher { + private val logger = new LoggerUtil(classOf[MonolithEnrolDispatcher]) + override def enrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = { + val req = new Request() + req.setRequestContext(ctx) + req.setRequestId("system") + req.setOperation("enrol") + req.put(JsonKey.USER_ID, userId); req.put(JsonKey.COURSE_ID, courseId); req.put(JsonKey.BATCH_ID, batchId) + val path = Option(ProjectUtil.getConfigValue("enrolment_actor_path")).filter(_.nonEmpty).getOrElse("/user/course-enrolment-actor") + context.actorSelection(path).tell(req, ActorRef.noSender) + logger.info(ctx, s"viewer.lp: enrol dispatched | user=$userId course=$courseId batch=$batchId mode=monolith") + } +} + +/** Distributed: POST the full enrol op to the enrolment service, authenticated with the system token. */ +class HttpEnrolDispatcher extends EnrolDispatcher { + private val logger = new LoggerUtil(classOf[HttpEnrolDispatcher]) + override def enrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = { + val base = Option(ProjectUtil.getConfigValue("enrolment_service_base_url")).filter(_.nonEmpty).getOrElse("http://lern-service:9000") + val body = s"""{"request":{"userId":"$userId","courseId":"$courseId","batchId":"$batchId"}}""" + val headers = new util.HashMap[String, String]() {{ + put("Content-Type", "application/json") + Option(ProjectUtil.getConfigValue("viewer_system_auth_token")).filter(_.nonEmpty) + .foreach(t => put("x-authenticated-user-token", t)) + }} + HttpClientUtil.post(base + "/v1/course/enroll", body, headers, ctx) + logger.info(ctx, s"viewer.lp: enrol dispatched | user=$userId course=$courseId batch=$batchId mode=distributed") + } +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala new file mode 100644 index 00000000..0e149384 --- /dev/null +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala @@ -0,0 +1,166 @@ +package org.sunbird.viewer.engine + +import org.apache.commons.collections4.CollectionUtils +import org.sunbird.activity.util.LpPolicyUtil +import org.sunbird.assessment.service.CassandraService +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.keys.JsonKey +import org.sunbird.logging.LoggerUtil +import org.sunbird.request.RequestContext +import org.sunbird.viewer.util.ProgressionPolicy + +import java.util +import scala.collection.JavaConverters._ + +/** LP progression: compute optionality once, open the next course(s) level-by-level via EnrolDispatcher, + * credit skills at completion. Pure decisions in ProgressionPolicy; caller supplies status + ancestorsOf. */ +class LpProgressionEngine(cassandraOperation: CassandraOperation, + enrolKeyspace: String, enrolTable: String, + lpPolicyUtil: LpPolicyUtil, + assessmentService: CassandraService, + dispatcher: EnrolDispatcher) { + + private val logger = new LoggerUtil(classOf[LpProgressionEngine]) + private val USER_SKILLS_TABLE = "user_skills" + + // status: (courseId,batchId)->status for all the user's enrolments; ancestorsOf: course->chain (root LAST). + def advance(userId: String, rootId: String, batchId: String, trackable: List[String], + status: Map[(String, String), Int], ancestorsOf: String => List[String], ctx: RequestContext): Unit = { + val childBatchOf = (c: String) => batchId + ":" + c + val courseComplete = (c: String) => status.get((c, childBatchOf(c))).contains(2) + + val levelByCourse = ProgressionPolicy.levelByCourse(trackable, ancestorsOf, rootId) // ancestorsOf runs once per course + // Halt only when optionality can't be established (Adaptive misconfigured: no pre-assessment) — open nothing. + if (!ensureOptionalityComputed(userId, rootId, batchId, trackable, levelByCourse, status, ctx)) return + val optional = readOptionalNodes(userId, rootId, batchId, ctx).toSet + val levels = ProgressionPolicy.orderedLevels(trackable, levelByCourse) + + // Level complete = all its required (non-optional) courses complete (empty required set = complete, §5). + def levelComplete(level: String): Boolean = + ProgressionPolicy.coursesOfLevel(level, trackable, levelByCourse).filterNot(optional.contains).forall(courseComplete) + + // Open first incomplete level: enrol all its optionals up front + the next single required course (§5). + levels.find(l => !levelComplete(l)).foreach { level => + val courses = ProgressionPolicy.coursesOfLevel(level, trackable, levelByCourse) + val nextRequired = courses.filterNot(optional.contains).find(c => !courseComplete(c)) + val toOpen = courses.filter(optional.contains) ++ nextRequired.toList + logger.info(ctx, s"viewer.lp: level opened | user=$userId root=$rootId level=$level open=[${toOpen.mkString(",")}]") + toOpen.foreach { c => + val childBatch = childBatchOf(c) + if (!status.contains((c, childBatch))) dispatcher.enrol(userId, c, childBatch, ctx) + else logger.info(ctx, s"viewer.lp: enrol skip(already) | user=$userId course=$c batch=$childBatch") + } + } + + // LP completion = every level complete -> credit durable skills (once; creditSkills no-ops if nothing new). + if (levels.nonEmpty && levels.forall(levelComplete)) { + logger.info(ctx, s"viewer.lp: complete | user=$userId root=$rootId") + creditSkills(userId, rootId, batchId, trackable, ctx) + } + } + + // Returns whether advance() may open courses; false ONLY for a misconfigured Adaptive LP (no pre-assessment). + private def ensureOptionalityComputed(userId: String, rootId: String, batchId: String, trackable: List[String], + levelByCourse: Map[String, String], + status: Map[(String, String), Int], ctx: RequestContext): Boolean = { + if (optionalityComputed(userId, rootId, batchId, ctx)) return true + val meta = lpPolicyUtil.lpMeta(rootId, ctx) + val policy = lpPolicyUtil.policyOf(meta) + if (policy.equalsIgnoreCase("Strict") || trackable.isEmpty) { + logger.info(ctx, s"viewer.lp: optionality computed(strict) optional=[] | user=$userId root=$rootId") + writeOptionalNodes(userId, rootId, batchId, Set.empty, ctx); return true + } + // pre-assessment = an assessment course in the FIRST level (seeds achieved skills before waivers). + val levels = ProgressionPolicy.orderedLevels(trackable, levelByCourse) + val preAssessment = ProgressionPolicy.coursesOfLevel(levels.headOption.getOrElse(""), trackable, levelByCourse) + .find(c => lpPolicyUtil.isAssessmentCourse(c, meta)) + if (policy.equalsIgnoreCase("Adaptive") && preAssessment.isEmpty) { + logger.warn(ctx, s"viewer.lp: Adaptive LP has no pre-assessment; halting (misconfigured, opening nothing) | user=$userId root=$rootId", null) + return false + } + val childBatchOf = (c: String) => batchId + ":" + c + val courseComplete = (c: String) => status.get((c, childBatchOf(c))).contains(2) + // Pre-assessment not done yet: don't compute waivers, but let advance open it (it's the first-level course). + if (preAssessment.exists(pa => !courseComplete(pa))) return true + val achieved = preAssessment.map(pa => skillsFromAssessment(userId, rootId, pa, batchId, ctx)).getOrElse(Set.empty) + // PriorLearning: a content course already completed under ANY batch is optional directly. + val completedCourses = status.collect { case ((c, _), 2) => c }.toSet // O(status) once, not O(trackable×status) + val priorCompleted = + if (policy.equalsIgnoreCase("PriorLearning")) trackable.filter(completedCourses.contains).toSet + else Set.empty[String] + val cMeta = lpPolicyUtil.courseMeta(trackable, meta) + val assessmentCourses = cMeta.collect { case (c, (_, true)) => c }.toSet + val skillsByCourse = cMeta.map { case (c, (s, _)) => c -> s } + logger.info(ctx, s"viewer.lp: optionality computing($policy) preAssess=${preAssessment.getOrElse("-")} achieved=${achieved.size} priorDone=${priorCompleted.size} | user=$userId root=$rootId") + writeOptionalNodes(userId, rootId, batchId, + ProgressionPolicy.computeOptionalNodes(policy, trackable, skillsByCourse, assessmentCourses, achieved, priorCompleted), ctx) + true + } + + private def isAssessmentCourse(rootId: String, courseId: String, ctx: RequestContext): Boolean = + lpPolicyUtil.isAssessmentCourse(courseId, lpPolicyUtil.lpMeta(rootId, ctx)) + + // terms of the questions the learner got fully correct in the pre-assessment's question set(s). + private def skillsFromAssessment(userId: String, rootId: String, courseId: String, batchId: String, ctx: RequestContext): Set[String] = { + val meta = lpPolicyUtil.lpMeta(rootId, ctx) + val childBatch = batchId + ":" + courseId + val correct = lpPolicyUtil.questionSetsOf(courseId, meta).flatMap { qs => + val attempts = assessmentService.getUserAssessments(userId, courseId, childBatch, qs, ctx) + if (attempts.isEmpty) Nil + else attempts.maxBy(_.totalScore).questions.collect { case q if q.maxScore > 0 && q.score == q.maxScore => q.questionId } + }.distinct + lpPolicyUtil.skillsOfQuestions(correct, meta) + } + + // Durable skill credit — ONCE at LP completion. Read-union-upsert (portable; no set-append needed). + private def creditSkills(userId: String, rootId: String, batchId: String, trackable: List[String], ctx: RequestContext): Unit = { + val earned = trackable.filter(c => isAssessmentCourse(rootId, c, ctx)).flatMap(c => skillsFromAssessment(userId, rootId, c, batchId, ctx)).toSet + if (earned.isEmpty) return + val existing = readUserSkills(userId, ctx) + val merged = existing ++ earned + if (merged.size == existing.size) return // nothing new — credit already granted + val row = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("skills", merged.asJava) }} + cassandraOperation.insertRecord(enrolKeyspace, USER_SKILLS_TABLE, row.asInstanceOf[util.Map[String, AnyRef]], ctx) + logger.info(ctx, s"LpProgressionEngine: credited ${earned.size} skills to user=$userId for LP=$rootId") + } + + private def writeOptionalNodes(userId: String, rootId: String, batchId: String, optional: Set[String], ctx: RequestContext): Unit = { + val selectMap = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", rootId); put("batchid", batchId) }} + val updateMap = new util.HashMap[String, AnyRef]() {{ put("optional_nodes", optional.asJava) }} + cassandraOperation.updateRecordV2(enrolKeyspace, enrolTable, selectMap, updateMap, true, ctx) + LpProgressionEngine.markOptionalityComputed(userId, rootId, batchId) // remember empty results too (no DB column) + } + + /** Computed? Non-empty optional_nodes is self-evident; an empty result is remembered in an in-process memo. */ + private def optionalityComputed(userId: String, rootId: String, batchId: String, ctx: RequestContext): Boolean = + readOptionalNodes(userId, rootId, batchId, ctx).nonEmpty || + LpProgressionEngine.isOptionalityComputed(userId, rootId, batchId) + + private def readOptionalNodes(userId: String, rootId: String, batchId: String, ctx: RequestContext): List[String] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", rootId); put("batchid", batchId) }} + val rows = cassandraOperation.getRecords(enrolKeyspace, enrolTable, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) + Option(rows.get(0).get("optional_nodes")).map(_.asInstanceOf[util.Collection[String]].asScala.toList).getOrElse(List()) + else List() + } + + private def readUserSkills(userId: String, ctx: RequestContext): Set[String] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} + val rows = cassandraOperation.getRecords(enrolKeyspace, USER_SKILLS_TABLE, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] + if (CollectionUtils.isNotEmpty(rows)) + Option(rows.get(0).get("skills")).map(_.asInstanceOf[util.Collection[String]].asScala.toSet).getOrElse(Set.empty) + else Set.empty + } +} + +object LpProgressionEngine { + // JVM-wide memo so empty-optionality isn't recomputed each pass. ponytail: unbounded; add a cap if it grows. + private val optionalityDone: java.util.Set[String] = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() + private def optKey(userId: String, rootId: String, batchId: String): String = s"$userId:$rootId:$batchId" + def markOptionalityComputed(userId: String, rootId: String, batchId: String): Unit = + optionalityDone.add(optKey(userId, rootId, batchId)) + def isOptionalityComputed(userId: String, rootId: String, batchId: String): Boolean = + optionalityDone.contains(optKey(userId, rootId, batchId)) +} diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala index 40cc576d..4150a269 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/util/ProgressionPolicy.scala @@ -24,6 +24,16 @@ object ProgressionPolicy { ancestorsOf: String => List[String], root: String): List[String] = trackable.flatMap(c => levelOf(c, ancestorsOf, root)).distinct + /** Precompute each course's level once so downstream lookups don't re-invoke ancestorsOf per call. */ + def levelByCourse(trackable: List[String], ancestorsOf: String => List[String], root: String): Map[String, String] = + trackable.flatMap(c => levelOf(c, ancestorsOf, root).map(c -> _)).toMap + + def orderedLevels(trackable: List[String], levelByCourse: Map[String, String]): List[String] = + trackable.flatMap(levelByCourse.get).distinct + + def coursesOfLevel(level: String, trackable: List[String], levelByCourse: Map[String, String]): List[String] = + trackable.filter(c => levelByCourse.get(c).contains(level)) + /** * A course is optional iff it is not an assessment and all of its (non-empty) skills are achieved. * `Strict` waives nothing; other policies differ only in how the caller builds `skillsAchieved`. diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala new file mode 100644 index 00000000..13b0eb78 --- /dev/null +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala @@ -0,0 +1,91 @@ +package org.sunbird.viewer.engine + +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.sunbird.activity.util.{LpMeta, LpPolicyUtil, NodeMeta} +import org.sunbird.assessment.service.CassandraService +import org.sunbird.cassandra.CassandraOperation +import org.sunbird.request.RequestContext +import org.sunbird.response.Response + +import java.util +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer + +class LpProgressionEngineSpec extends AnyFlatSpec with Matchers with MockFactory { + + private val ctx = new RequestContext() + + // 2-level LP: crsA under L1, crsB under L2 (ancestors nearest-first, root LAST). + private val ancestorsOf: String => List[String] = { + case "crsA" => List("L1", "lp") + case "crsB" => List("L2", "lp") + case _ => Nil + } + private val trackable = List("crsA", "crsB") + + private def emptyRows: Response = { val r = new Response(); r.put("response", new util.ArrayList[util.Map[String, AnyRef]]()); r } + private def optionalRows(opt: String*): Response = { + val row = new util.HashMap[String, AnyRef](); row.put("optional_nodes", opt.toList.asJava) + val list = new util.ArrayList[util.Map[String, AnyRef]](); list.add(row) + val r = new Response(); r.put("response", list); r + } + + /** Fake transport: records (user, course, batch) instead of messaging/HTTP. */ + private class FakeDispatcher extends EnrolDispatcher { + val enrolled: ListBuffer[(String, String, String)] = ListBuffer.empty + override def enrol(userId: String, courseId: String, batchId: String, ctx: RequestContext): Unit = + enrolled += ((userId, courseId, batchId)) + } + + private def engineWith(ops: CassandraOperation, lp: LpPolicyUtil, disp: EnrolDispatcher): LpProgressionEngine = + new LpProgressionEngine(ops, "ks", "user_enrolments", lp, new CassandraService(Some(ops)), disp) + + "advance (Strict)" should "open the first level's required course and gate the next level" in { + val ops = mock[CassandraOperation] + val lp = mock[LpPolicyUtil] + val disp = new FakeDispatcher + // optional_nodes read -> empty (not yet computed); Strict writes an empty optional set. + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows).anyNumberOfTimes() + (ops.updateRecordV2(_: String, _: String, _: util.Map[String, AnyRef], _: util.Map[String, AnyRef], _: Boolean, _: RequestContext)) + .expects(*, *, *, *, *, *).returns(new Response()).anyNumberOfTimes() + (lp.lpMeta(_: String, _: RequestContext)).expects(*, *).returns(LpMeta("Strict", "", "", Map.empty[String, NodeMeta])).anyNumberOfTimes() + (lp.policyOf(_: LpMeta)).expects(*).returns("Strict").anyNumberOfTimes() + + engineWith(ops, lp, disp).advance("uA", "lp", "bA", trackable, Map.empty, ancestorsOf, ctx) + + disp.enrolled.toList shouldBe List(("uA", "crsA", "bA:crsA")) // first level's course only; crsB (L2) gated + } + + "advance" should "skip a waived course and open the next level's course" in { + val ops = mock[CassandraOperation] + val lp = mock[LpPolicyUtil] + val disp = new FakeDispatcher + // optional_nodes already = [crsA] -> optionality is 'computed' (non-empty), so ensureOptionality + // returns immediately; crsA is treated as waived, L1 is complete, L2's crsB opens. + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(optionalRows("crsA")).anyNumberOfTimes() + + engineWith(ops, lp, disp).advance("uB", "lp", "bB", trackable, Map.empty, ancestorsOf, ctx) + + disp.enrolled.toList shouldBe List(("uB", "crsB", "bB:crsB")) // crsA waived -> next required course opens + } + + "advance (Adaptive, no pre-assessment)" should "halt and open nothing (misconfigured)" in { + val ops = mock[CassandraOperation] + val lp = mock[LpPolicyUtil] + val disp = new FakeDispatcher + // optionality not yet computed; Adaptive policy; no course is an assessment -> no pre-assessment -> halt. + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows).anyNumberOfTimes() + (lp.lpMeta(_: String, _: RequestContext)).expects(*, *).returns(LpMeta("Adaptive", "", "", Map.empty[String, NodeMeta])).anyNumberOfTimes() + (lp.policyOf(_: LpMeta)).expects(*).returns("Adaptive").anyNumberOfTimes() + (lp.isAssessmentCourse(_: String, _: LpMeta)).expects(*, *).returns(false).anyNumberOfTimes() + + engineWith(ops, lp, disp).advance("uC", "lp", "bC", trackable, Map.empty, ancestorsOf, ctx) + + disp.enrolled shouldBe empty // misconfigured Adaptive LP opens no course (no updateRecordV2 either) + } +} diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala index 50752059..01169354 100644 --- a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/util/ProgressionPolicySpec.scala @@ -43,6 +43,15 @@ class ProgressionPolicySpec extends AnyFlatSpec with Matchers { ProgressionPolicy.orderedLevels(order, ancestorsOf, "do_lp") shouldBe List("L1", "L2", "L3", "L4") } + "levelByCourse (precomputed map)" should "match the ancestorsOf-based overloads" in { + val m = ProgressionPolicy.levelByCourse(order, ancestorsOf, "do_lp") + m shouldBe Map("CRS-A" -> "L1", "CRS-B" -> "L2", "CRS-C" -> "L2", + "CRS-D" -> "L3", "CRS-E" -> "L3", "CRS-F" -> "L4") + ProgressionPolicy.orderedLevels(order, m) shouldBe ProgressionPolicy.orderedLevels(order, ancestorsOf, "do_lp") + ProgressionPolicy.coursesOfLevel("L2", order, m) shouldBe + ProgressionPolicy.coursesOfLevel("L2", order, ancestorsOf, "do_lp") + } + "computeOptionalNodes" should "waive a fully-known non-assessment course but never an assessment" in { val opt = ProgressionPolicy.computeOptionalNodes( policy = "Adaptive", From 7e50911704b930c3cf0deeba2e3a33949dede42a Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Wed, 12 Aug 2026 17:14:16 +0530 Subject: [PATCH 29/30] feat(viewer): LP course-completion trigger, parent-LP resolution, course cert toggle - Bridge to the LP root only on a course's not-complete->complete transition (writeAllNodeEnrolments returns the transitioned set; gate the bridge on it). - Resolve the parent LP from the authoritative course_batch (by batchid), picking the row whose courseid differs from the completed course; extracted to a pure resolveParentLp for testability. Standalone (colon-free) batch -> no trigger. - course_certificate_enabled toggle (default true) gating per-course certs; LP cert unaffected. Root progress/status + LP cert on LP completion in the engine. - Tests: resolveParentLp (standalone/child/self-only/tiebreak) + parseCourseCertEnabled. --- .../viewer/actor/ViewerAggregatorActor.scala | 73 +++++++++++++++++-- .../viewer/engine/LpProgressionEngine.scala | 31 +++++++- .../actor/ViewerAggregatorActorTest.scala | 43 +++++++++++ .../engine/LpProgressionEngineSpec.scala | 42 +++++++++-- 4 files changed, 173 insertions(+), 16 deletions(-) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index 361ca914..c037685e 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -41,12 +41,17 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { private val enrolmentDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_COURSE_DB) private val activityAggDBInfo = Util.dbInfoMap.get(JsonKey.GROUP_ACTIVITY_DB) private val consumptionDBInfo = Util.dbInfoMap.get(JsonKey.LEARNER_CONTENT_DB) + private val courseBatchDBInfo = Util.dbInfoMap.get(JsonKey.COURSE_BATCH_DB) private val CONSUMPTION_TABLE = "user_content_consumption" + // Course-level certificates on course completion (the LP cert is always issued by the engine). + // Default true; set course_certificate_enabled=false to suppress course certs only. + private val courseCertEnabled: Boolean = + ViewerAggregatorActor.parseCourseCertEnabled(ProjectUtil.getConfigValue("course_certificate_enabled")) // LP progression extracted to a focused, injectable engine (SRP); transport behind a dispatcher (OCP). // lazy so `context` is set by the time they initialize. private lazy val enrolDispatcher: org.sunbird.viewer.engine.EnrolDispatcher = org.sunbird.viewer.engine.EnrolDispatcher(context) private lazy val lpEngine = new org.sunbird.viewer.engine.LpProgressionEngine( - cassandraOperation, enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, lpPolicyUtil, assessmentService, enrolDispatcher) + cassandraOperation, enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, lpPolicyUtil, assessmentService, enrolDispatcher, certificateUtil) override def onReceive(request: Request): Unit = { request.getOperation match { @@ -128,10 +133,43 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // 7. Update user_enrolments status for EVERY enrolled node in this tree (approach #1: key off the // child enrolment rows that already exist; root included). Cert fires once, on transition to complete. - writeAllNodeEnrolments(userId, courseId, batchId, nodeProgress.toMap, contentStatusMap, ctx) + // Returns the node ids that transitioned to complete (status != 2 -> 2) in THIS pass. + val completedNow = writeAllNodeEnrolments(userId, courseId, batchId, nodeProgress.toMap, contentStatusMap, ctx) - // 8. LP progression (only when this root is an LP): optionality once, open next course(s), credit at completion. + // 8. LP progression: for the LP root, advance. For a chained child, bridge to the LP root ONLY when the + // child course just COMPLETED this pass (not on every partial view) — the completion is the trigger. if (trackable.nonEmpty) advanceLp(userId, courseId, batchId, trackable, ctx) + else if (completedNow.contains(courseId)) bridgeToRoot(userId, courseId, batchId, ctx) + } + + /** A child course just completed -> if it belongs to an LP, re-fire the LP-root aggregate so it re-advances. */ + private def bridgeToRoot(userId: String, courseId: String, courseBatchId: String, ctx: RequestContext): Unit = + parentLpOf(courseId, courseBatchId, ctx).foreach { case (lpId, lpBatch) => + val req = new Request(); req.setRequestContext(ctx); req.setOperation("aggregate") + req.put(JsonKey.USER_ID, userId); req.put("courseId", lpId); req.put("batchId", lpBatch) + self.tell(req, org.apache.pekko.actor.ActorRef.noSender) + logger.info(ctx, s"viewer.rollup: child->LP bridge | user=$userId course=$courseId childBatch=$courseBatchId lp=$lpId lpBatch=$lpBatch") + } + + /** + * Parent LP of a just-completed course, or None for a standalone course. Resolves from the authoritative + * structural map (course_batch): strip the ":" prefix to get the LP batch, read course_batch by batchid, + * and pick the row whose courseid differs from the completed course (that's the LP). Requires the + * course_batch(batchid) secondary index (see migrations) so this is a keyed read, not a scan — same + * pattern as user_enrolments_by_batch. + * ponytail: leans on the ":" child-batch convention (rootBatch:childId) to recover the LP batch id. + * Ceiling: assumes no standalone batch id contains ":". Upgrade: stamp parent_collection_id/ + * parent_context_id on the course_batch/enrolment row at creation and read those directly (migration-time). + */ + private def parentLpOf(courseId: String, courseBatchId: String, ctx: RequestContext): Option[(String, String)] = + ViewerAggregatorActor.resolveParentLp(courseId, courseBatchId, lpBatch => courseBatchRowsByBatchId(lpBatch, ctx)) + + /** course_batch rows for a batch id — keyed read via the course_batch(batchid) secondary index. */ + private def courseBatchRowsByBatchId(batchId: String, ctx: RequestContext): util.List[util.Map[String, AnyRef]] = { + val filters = new util.HashMap[String, AnyRef]() {{ put("batchid", batchId) }} + cassandraOperation.getRecords(courseBatchDBInfo.getKeySpace, courseBatchDBInfo.getTableName, + filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) + .getResult.getOrDefault(JsonKey.RESPONSE, new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]] } private def completedCountOf(a: UserEnrolmentAgg): Int = @@ -184,7 +222,8 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { */ private def writeAllNodeEnrolments(userId: String, rootId: String, batchId: String, nodeProgress: Map[String, (Int, List[String])], - contentStatusMap: Map[String, ContentStatus], ctx: RequestContext): Unit = { + contentStatusMap: Map[String, ContentStatus], ctx: RequestContext): Set[String] = { + val completedNow = scala.collection.mutable.Set[String]() val filters = new util.HashMap[String, AnyRef]() {{ put("userid", userId) }} val enrolRows = cassandraOperation.getRecords(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, filters.asInstanceOf[util.Map[String, AnyRef]], null, ctx) @@ -219,11 +258,15 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { }} cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) if (status == 2 && currentStatus != 2) { - logger.info(ctx, s"viewer.rollup: node completed -> cert | user=$userId course=$nodeId batch=$nodeCtx") - certificateUtil.publishCertificateIssueEvent(userId, nodeId, nodeCtx, ctx) + completedNow += nodeId + if (courseCertEnabled) { + logger.info(ctx, s"viewer.rollup: node completed -> cert | user=$userId course=$nodeId batch=$nodeCtx") + certificateUtil.publishCertificateIssueEvent(userId, nodeId, nodeCtx, ctx) + } else logger.info(ctx, s"viewer.rollup: node completed, course cert suppressed (course_certificate_enabled=false) | user=$userId course=$nodeId batch=$nodeCtx") } } } + completedNow.toSet } /** @@ -278,4 +321,22 @@ object ViewerAggregatorActor { fresh.foreach { case (k, v) => merged.put(k, v) } merged } + + // Parent LP of a completed course: strip the ":" child-batch prefix to get the LP batch, read course_batch + // by that batch id, and pick the row whose courseid differs from the completed course (a course may hold + // its own records under the same batch string; only a DIFFERENT courseid is the parent LP). None for a + // standalone (colon-free) batch — fetchByBatchId is not invoked in that case. + private[actor] def resolveParentLp(courseId: String, courseBatchId: String, + fetchByBatchId: String => java.util.List[java.util.Map[String, AnyRef]]): Option[(String, String)] = { + if (courseBatchId == null || !courseBatchId.contains(":")) None + else { + val lpBatch = courseBatchId.substring(0, courseBatchId.indexOf(":")) + fetchByBatchId(lpBatch).asScala + .flatMap(r => Option(r.get("courseId")).map(_.toString)) + .find(_ != courseId).map(lpId => (lpId, lpBatch)) + } + } + + // Course-cert toggle: default true; only the literal "false" disables it (the LP cert is unaffected). + private[actor] def parseCourseCertEnabled(cfg: String): Boolean = !"false".equalsIgnoreCase(Option(cfg).getOrElse("")) } diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala index 0e149384..ae6cd46c 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/engine/LpProgressionEngine.scala @@ -1,7 +1,7 @@ package org.sunbird.viewer.engine import org.apache.commons.collections4.CollectionUtils -import org.sunbird.activity.util.LpPolicyUtil +import org.sunbird.activity.util.{CertificateUtil, LpPolicyUtil} import org.sunbird.assessment.service.CassandraService import org.sunbird.cassandra.CassandraOperation import org.sunbird.keys.JsonKey @@ -18,7 +18,8 @@ class LpProgressionEngine(cassandraOperation: CassandraOperation, enrolKeyspace: String, enrolTable: String, lpPolicyUtil: LpPolicyUtil, assessmentService: CassandraService, - dispatcher: EnrolDispatcher) { + dispatcher: EnrolDispatcher, + certificateUtil: CertificateUtil) { private val logger = new LoggerUtil(classOf[LpProgressionEngine]) private val USER_SKILLS_TABLE = "user_skills" @@ -52,13 +53,35 @@ class LpProgressionEngine(cassandraOperation: CassandraOperation, } } - // LP completion = every level complete -> credit durable skills (once; creditSkills no-ops if nothing new). - if (levels.nonEmpty && levels.forall(levelComplete)) { + // Root progress/status/cert from trackable-course completion: the root's own leaf consumption is empty + // (leaves are consumed under child batches), so derive its progress from how many required courses are done. + val requiredCourses = trackable.filterNot(optional.contains) + val allComplete = levels.nonEmpty && levels.forall(levelComplete) + writeRootProgress(userId, rootId, batchId, requiredCourses.count(courseComplete), requiredCourses.size, + allComplete, status.get((rootId, batchId)).getOrElse(0), ctx) + + if (allComplete) { logger.info(ctx, s"viewer.lp: complete | user=$userId root=$rootId") creditSkills(userId, rootId, batchId, trackable, ctx) } } + // Update the LP-root user_enrolments row (progress/%/status; completedon on the 2-transition) and fire the + // LP certificate once when status first reaches 2. Partial update — does not touch contentstatus. + private def writeRootProgress(userId: String, rootId: String, batchId: String, done: Int, total: Int, + allComplete: Boolean, currentStatus: Int, ctx: RequestContext): Unit = { + val pct = if (total <= 0) 100 else math.min(100, done * 100 / total) + val rootStatus = if (allComplete) 2 else if (done > 0) 1 else 0 + val select = new util.HashMap[String, AnyRef]() {{ put("userid", userId); put("courseid", rootId); put("batchid", batchId) }} + val update = new util.HashMap[String, AnyRef]() {{ + put("progress", Integer.valueOf(done)); put("completionpercentage", Integer.valueOf(pct)); put("status", Integer.valueOf(rootStatus)) + if (rootStatus == 2 && currentStatus != 2) put("completedon", new java.util.Date()) + }} + cassandraOperation.updateRecordV2(enrolKeyspace, enrolTable, select, update, true, ctx) + logger.info(ctx, s"viewer.lp: root progress | user=$userId root=$rootId done=$done/$total pct=$pct status=$rootStatus") + if (rootStatus == 2 && currentStatus != 2) certificateUtil.publishCertificateIssueEvent(userId, rootId, batchId, ctx) + } + // Returns whether advance() may open courses; false ONLY for a misconfigured Adaptive LP (no pre-assessment). private def ensureOptionalityComputed(userId: String, rootId: String, batchId: String, trackable: List[String], levelByCourse: Map[String, String], diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala index 0113d3e6..80a5dc39 100644 --- a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/actor/ViewerAggregatorActorTest.scala @@ -89,4 +89,47 @@ class ViewerAggregatorActorTest extends AnyFlatSpec with Matchers with MockFacto merged.get("leaf-a") shouldBe Integer.valueOf(1) merged.size() shouldBe 1 } + + // resolveParentLp: the completion trigger's parent-lookup. Child batch = "LPbatch:childId"; the LP is the + // course_batch row keyed by the stripped LPbatch whose courseid differs from the completed course. + private def batchRow(courseId: String): util.Map[String, AnyRef] = + new util.HashMap[String, AnyRef]() {{ put("courseId", courseId) }} + private def batchRows(courseIds: String*): util.List[util.Map[String, AnyRef]] = { + val l = new util.ArrayList[util.Map[String, AnyRef]](); courseIds.foreach(c => l.add(batchRow(c))); l + } + + "resolveParentLp" should "return None for a standalone (colon-free) batch without fetching" in { + var fetched = false + val out = ViewerAggregatorActor.resolveParentLp("course-1", "plainBatch", _ => { fetched = true; batchRows() }) + out shouldBe None + fetched shouldBe false // standalone course -> no course_batch read, no trigger + } + + "resolveParentLp" should "resolve the LP course + stripped LP batch from the child batch" in { + val out = ViewerAggregatorActor.resolveParentLp( + "course-1", "lpBatch-9:course-1", lpBatch => { lpBatch shouldBe "lpBatch-9"; batchRows("course-1", "lp-root") }) + out shouldBe Some(("lp-root", "lpBatch-9")) // the row whose courseid != the completed course + } + + "resolveParentLp" should "return None when the batch maps only to the course itself" in { + // course consumed under its own colon-batch, but course_batch has no differing (LP) row -> not an LP child + val out = ViewerAggregatorActor.resolveParentLp("course-1", "lpBatch-9:course-1", _ => batchRows("course-1")) + out shouldBe None + } + + "resolveParentLp" should "pick the first differing course id when several rows share the LP batch" in { + // In the live schema only the LP-root row is keyed by the bare lpBatch (children are lpBatch:childId), so + // this is a defensive/bad-data case: with multiple differing rows the resolver is first-match-wins. + val out = ViewerAggregatorActor.resolveParentLp("course-1", "lpBatch-9:course-1", + _ => batchRows("course-1", "lp-root", "lp-other")) + out shouldBe Some(("lp-root", "lpBatch-9")) + } + + // parseCourseCertEnabled: course certs default ON; only literal "false" disables (LP cert unaffected). + "parseCourseCertEnabled" should "default to true when unset/blank and honor an explicit false" in { + ViewerAggregatorActor.parseCourseCertEnabled(null) shouldBe true + ViewerAggregatorActor.parseCourseCertEnabled("") shouldBe true + ViewerAggregatorActor.parseCourseCertEnabled("true") shouldBe true + ViewerAggregatorActor.parseCourseCertEnabled("FALSE") shouldBe false + } } diff --git a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala index 13b0eb78..6f487de5 100644 --- a/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala +++ b/modules/viewer/actors/src/test/scala/org/sunbird/viewer/engine/LpProgressionEngineSpec.scala @@ -3,7 +3,7 @@ package org.sunbird.viewer.engine import org.scalamock.scalatest.MockFactory import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import org.sunbird.activity.util.{LpMeta, LpPolicyUtil, NodeMeta} +import org.sunbird.activity.util.{CertificateUtil, LpMeta, LpPolicyUtil, NodeMeta} import org.sunbird.assessment.service.CassandraService import org.sunbird.cassandra.CassandraOperation import org.sunbird.request.RequestContext @@ -39,13 +39,19 @@ class LpProgressionEngineSpec extends AnyFlatSpec with Matchers with MockFactory enrolled += ((userId, courseId, batchId)) } - private def engineWith(ops: CassandraOperation, lp: LpPolicyUtil, disp: EnrolDispatcher): LpProgressionEngine = - new LpProgressionEngine(ops, "ks", "user_enrolments", lp, new CassandraService(Some(ops)), disp) + private def engineWith(ops: CassandraOperation, lp: LpPolicyUtil, disp: EnrolDispatcher, cert: CertificateUtil): LpProgressionEngine = + new LpProgressionEngine(ops, "ks", "user_enrolments", lp, new CassandraService(Some(ops)), disp, cert) + + // updateRecordV2 stub (writeOptionalNodes + writeRootProgress) — accept any, return a Response. + private def stubUpdate(ops: CassandraOperation): Unit = + (ops.updateRecordV2(_: String, _: String, _: util.Map[String, AnyRef], _: util.Map[String, AnyRef], _: Boolean, _: RequestContext)) + .expects(*, *, *, *, *, *).returns(new Response()).anyNumberOfTimes() "advance (Strict)" should "open the first level's required course and gate the next level" in { val ops = mock[CassandraOperation] val lp = mock[LpPolicyUtil] val disp = new FakeDispatcher + val cert = mock[CertificateUtil] // optional_nodes read -> empty (not yet computed); Strict writes an empty optional set. (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) .expects(*, *, *, *, *).returns(emptyRows).anyNumberOfTimes() @@ -54,7 +60,7 @@ class LpProgressionEngineSpec extends AnyFlatSpec with Matchers with MockFactory (lp.lpMeta(_: String, _: RequestContext)).expects(*, *).returns(LpMeta("Strict", "", "", Map.empty[String, NodeMeta])).anyNumberOfTimes() (lp.policyOf(_: LpMeta)).expects(*).returns("Strict").anyNumberOfTimes() - engineWith(ops, lp, disp).advance("uA", "lp", "bA", trackable, Map.empty, ancestorsOf, ctx) + engineWith(ops, lp, disp, cert).advance("uA", "lp", "bA", trackable, Map.empty, ancestorsOf, ctx) disp.enrolled.toList shouldBe List(("uA", "crsA", "bA:crsA")) // first level's course only; crsB (L2) gated } @@ -63,20 +69,44 @@ class LpProgressionEngineSpec extends AnyFlatSpec with Matchers with MockFactory val ops = mock[CassandraOperation] val lp = mock[LpPolicyUtil] val disp = new FakeDispatcher + val cert = mock[CertificateUtil] // optional_nodes already = [crsA] -> optionality is 'computed' (non-empty), so ensureOptionality // returns immediately; crsA is treated as waived, L1 is complete, L2's crsB opens. (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) .expects(*, *, *, *, *).returns(optionalRows("crsA")).anyNumberOfTimes() + stubUpdate(ops) - engineWith(ops, lp, disp).advance("uB", "lp", "bB", trackable, Map.empty, ancestorsOf, ctx) + engineWith(ops, lp, disp, cert).advance("uB", "lp", "bB", trackable, Map.empty, ancestorsOf, ctx) disp.enrolled.toList shouldBe List(("uB", "crsB", "bB:crsB")) // crsA waived -> next required course opens } + "advance" should "mark the LP root complete + fire the cert when all required courses are done" in { + val ops = mock[CassandraOperation] + val lp = mock[LpPolicyUtil] + val disp = new FakeDispatcher + val cert = mock[CertificateUtil] + // single-course LP; the course is already complete in the snapshot -> root reaches 100%/status=2. + val oneCourse = List("crsA") + val statusDone = Map(("crsA", "bD:crsA") -> 2) // root (lp,bD) absent -> currentStatus 0 -> cert fires once + (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) + .expects(*, *, *, *, *).returns(emptyRows).anyNumberOfTimes() + (lp.lpMeta(_: String, _: RequestContext)).expects(*, *).returns(LpMeta("Strict", "", "", Map.empty[String, NodeMeta])).anyNumberOfTimes() + (lp.policyOf(_: LpMeta)).expects(*).returns("Strict").anyNumberOfTimes() + (lp.isAssessmentCourse(_: String, _: LpMeta)).expects(*, *).returns(false).anyNumberOfTimes() // creditSkills -> no assessment courses + stubUpdate(ops) + (cert.publishCertificateIssueEvent(_: String, _: String, _: String, _: RequestContext)).expects("uD", "lp", "bD", *).once() + + engineWith(ops, lp, disp, cert).advance("uD", "lp", "bD", oneCourse, statusDone, ancestorsOf, ctx) + + disp.enrolled shouldBe empty // nothing left to open; LP root completed + } + "advance (Adaptive, no pre-assessment)" should "halt and open nothing (misconfigured)" in { val ops = mock[CassandraOperation] val lp = mock[LpPolicyUtil] val disp = new FakeDispatcher + val cert = mock[CertificateUtil] // optionality not yet computed; Adaptive policy; no course is an assessment -> no pre-assessment -> halt. (ops.getRecords(_: String, _: String, _: util.Map[String, AnyRef], _: util.List[String], _: RequestContext)) .expects(*, *, *, *, *).returns(emptyRows).anyNumberOfTimes() @@ -84,7 +114,7 @@ class LpProgressionEngineSpec extends AnyFlatSpec with Matchers with MockFactory (lp.policyOf(_: LpMeta)).expects(*).returns("Adaptive").anyNumberOfTimes() (lp.isAssessmentCourse(_: String, _: LpMeta)).expects(*, *).returns(false).anyNumberOfTimes() - engineWith(ops, lp, disp).advance("uC", "lp", "bC", trackable, Map.empty, ancestorsOf, ctx) + engineWith(ops, lp, disp, cert).advance("uC", "lp", "bC", trackable, Map.empty, ancestorsOf, ctx) disp.enrolled shouldBe empty // misconfigured Adaptive LP opens no course (no updateRecordV2 either) } From cf33771903199c347f407032261847de5304d8e6 Mon Sep 17 00:00:00 2001 From: Aiman Sharief Date: Wed, 12 Aug 2026 18:29:19 +0530 Subject: [PATCH 30/30] fix: Advance LP for complete courses; issue cert only on transition Update ViewerAggregatorActor so completedNow collects nodes whose status == 2 after the write pass (not only nodes that transitioned this pass). This makes LP bridging/advance run for already-complete courses (idempotent), while certificate issuance remains gated to the actual transition (status != 2 -> 2) to avoid duplicate cert events. Clarify related comments to reflect the changed semantics. --- .../viewer/actor/ViewerAggregatorActor.scala | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala index c037685e..a2dba3b2 100644 --- a/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala +++ b/modules/viewer/actors/src/main/scala/org/sunbird/viewer/actor/ViewerAggregatorActor.scala @@ -133,11 +133,12 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { // 7. Update user_enrolments status for EVERY enrolled node in this tree (approach #1: key off the // child enrolment rows that already exist; root included). Cert fires once, on transition to complete. - // Returns the node ids that transitioned to complete (status != 2 -> 2) in THIS pass. + // Returns the node ids that are COMPLETE (status == 2) after this pass. val completedNow = writeAllNodeEnrolments(userId, courseId, batchId, nodeProgress.toMap, contentStatusMap, ctx) - // 8. LP progression: for the LP root, advance. For a chained child, bridge to the LP root ONLY when the - // child course just COMPLETED this pass (not on every partial view) — the completion is the trigger. + // 8. LP progression: for the LP root, advance. For a chained child, bridge to the LP root when the child + // course is COMPLETE this pass (not on a partial view) — the completion is the trigger. Idempotent: + // a re-submit of an already-complete course still re-advances the LP. if (trackable.nonEmpty) advanceLp(userId, courseId, batchId, trackable, ctx) else if (completedNow.contains(courseId)) bridgeToRoot(userId, courseId, batchId, ctx) } @@ -257,8 +258,13 @@ class ViewerAggregatorActor extends BaseEnrolmentActor { if (status == 2 && currentStatus != 2) put("completedon", new java.util.Date()) }} cassandraOperation.updateRecordV2(enrolmentDBInfo.getKeySpace, enrolmentDBInfo.getTableName, selectMap, updateMap, true, ctx) + // Complete after this pass -> eligible to bridge to the LP. Gate on COMPLETENESS (status==2), not + // the status!=2->2 transition: a course that was already complete (re-submit, or completed before + // the bridge existed) must still re-advance the LP. advanceLp is idempotent, so re-firing is safe; + // a partial pass (status 1) never bridges. + if (status == 2) completedNow += nodeId + // Cert fires once, only on the transition to complete (avoid re-issuing on a re-submit). if (status == 2 && currentStatus != 2) { - completedNow += nodeId if (courseCertEnabled) { logger.info(ctx, s"viewer.rollup: node completed -> cert | user=$userId course=$nodeId batch=$nodeCtx") certificateUtil.publishCertificateIssueEvent(userId, nodeId, nodeCtx, ctx)